@proceedings {2017:benchflow:bpmds, title = {Performance Comparison Between BPMN 2.0 Workflow Management Systems Versions}, year = {2017}, month = {June}, publisher = {Springer}, address = {Essen, Germany}, abstract = {Software has become a rapidly evolving artifact and Workflow Management Systems (WfMSs) are not an exception. WfMSs{\textquoteright} changes may impact key performance indicators or resource consumption levels may change among different versions. Thus, users considering a WfMS upgrade need to evaluate the extent of such changes for frequently issued workload. Deriving such information requires running performance experiments with appropriate workloads. In this paper, we propose a novel method for deriving a structurally representative workload from a given business process collection, which we later use to evaluate the performance and resource consumption over four versions of two open-source WfMSs, for different numbers of simulated users. In our case study scenario the results reveal relevant variations in the WfMSs{\textquoteright} performance and resource consumption, indicating a decrease in performance for newer versions.}, keywords = {BenchFlow, BPMN, Performance Regression, Performance Testing, workflow engine, Workflow Management Systems}, author = {Vincenzo Ferme and Marigianna Skouradaki and Ana Ivanchikj and Cesare Pautasso and Frank Leymann} } @conference {benchflow:2016:closer, title = {A Container-centric Methodology for Benchmarking Workflow Management Systems}, booktitle = {6th International Conference on Cloud Computing and Service Science (CLOSER 2016)}, year = {2016}, month = {April}, pages = {74-84}, publisher = {SciTePress}, organization = {SciTePress}, address = {Rome, Italy}, abstract = {Trusted benchmarks should provide reproducible results obtained following a transparent and well-defined process. In this paper, we show how Containers, originally developed to ease the automated deployment of Cloud application components, can be used in the context of a benchmarking methodology. The proposed methodology focuses on Workflow Management Systems (WfMSs), a critical service orchestration middleware, which can be characterized by its architectural complexity, for which Docker Containers offer a highly suitable approach. The contributions of our work are: 1) a new benchmarking approach taking full advantage of containerization technologies; and 2) the formalization of the interaction process with the WfMS vendors described clearly in a written agreement. Thus, we take advantage of emerging Cloud technologies to address technical challenges, ensuring the performance measurements can be trusted. We also make the benchmarking process transparent, automated, and repeatable so that WfMS vendors can join the benchmarking effort.}, keywords = {BenchFlow, benchmarking, Docker}, doi = {10.5220/0005908400740084}, author = {Vincenzo Ferme and Ana Ivanchikj and Cesare Pautasso and Marigianna Skouradaki and Frank Leymann} } @conference {benchflow:2016:caise, title = {Micro-Benchmarking BPMN 2.0 Workflow Management Systems with Workflow Patterns}, booktitle = {Proc. of the 28th International Conference on Advanced Information Systems Engineering (CAISE)}, year = {2016}, month = {June}, pages = {67--82}, publisher = {Springer}, organization = {Springer}, address = {Ljubljana, Slovenia}, abstract = {Although Workflow Management Systems (WfMSs) are a key component in workflow technology, research work for assessing and comparing their performance is limited. This work proposes the first micro- benchmark for WfMSs that can execute BPMN 2.0 workflows. To this end, we focus on studying the performance impact of well-known workflow patterns expressed in BPMN 2.0 with respect to three open source WfMSs (i.e., Activiti, jBPM and Camunda). We executed all the experiments under a reliable environment and produced a set of meaningful metrics. This paper contributes to the area of workflow technology by defining building blocks for more complex BPMN 2.0 WfMS benchmarks. The results have shown bottlenecks on architectural design decisions, resource utilization, and limits on the load a WfMS can sustain, especially for the cases of complex and parallel structures. Experiments on a mix of workflow patterns indicated that there are no unexpected performance side effects when executing different workflow patterns concurrently, although the duration of the individual workflows that comprised the mix was increased.}, keywords = {BenchFlow, benchmarking, BPMN, Microbenchmark, workflow engine, Workflow Management Systems, workflow patterns}, doi = {10.1007/978-3-319-39696-5_5}, url = {http://www2.informatik.uni-stuttgart.de/cgi-bin/NCSTRL/NCSTRL_view.pl?id=INPROC-2016-05\&engl=0}, author = {Marigianna Skouradaki and Vincenzo Ferme and Cesare Pautasso and Frank Leymann and Andr{\'e} van Hoorn} } @conference {benchflow:2015:closer, title = {"BPELanon": Protect Business Processes on the Cloud}, booktitle = {5th International Conference on Cloud Computing and Service Science (CLOSER 2015)}, year = {2015}, month = {May}, pages = {241-250}, publisher = {SciTePress}, organization = {SciTePress}, address = {Lisbon, Portugal}, abstract = {The advent of Cloud computing supports the offering of many Business Process Management applications on a distributed, per-use basis environment through its infrastructure. Due to the fact that privacy is still an open issue in the Cloud, many companies are reluctant to move their Business Processes on a public Cloud. Since the Cloud environment can be beneficiary for the Business Processes, the investigation of privacy issues needs to be further examined. In order to enforce the Business Process sharing on the Cloud we propose a methodology ({\textquotedblleft}BPELanon{\textquotedblright}) for the anonymization of Business Processes expressed in the Web Service Business Process Execution Language (BPEL). The method transforms a process to preserve its original structure and run-time behavior, while completely anonymizing its business semantics. In this work we set the theoretical framework of the method and define a five management layers architecture to support its realization. We developed a tool that implements the {\textquotedblleft}BPELanon{\textquotedblright} method, validate its functionality and evaluate its performance against a collection of real-world process models that were conducted in the scope of research projects.}, keywords = {Anonymization, BenchFlow, BPEL}, doi = {10.5220/0005427502410250}, author = {Marigianna Skouradaki and Vincenzo Ferme and Frank Leymann and Cesare Pautasso and Dieter Roller} } @conference {benchflow:2015:icpe, title = {On the Road to Benchmarking BPMN 2.0 Workflow Engines}, booktitle = {6th ACM/SPEC International Conference on Performance Engineering}, year = {2015}, month = {January}, publisher = {IEEE}, organization = {IEEE}, address = {Austin, TX, USA}, abstract = {Workflow Management Systems (WfMSs) provide platforms for delivering complex service-oriented applications that need to satisfy enterprise-grade quality of service requirements such as dependability and scalability. In this paper we focus on the case of benchmarking the performance of the core of WfMSs, Workflow Engines, that are compliant with the Business Process Model and Notation 2.0 (BPMN 2.0) standard. We first explore the main challenges that need to be met when designing such a benchmark and describe the approaches we designed for tackling them in the BenchFlow project. We discuss our approach to distill the essence of real-world processes to create from it processes for the benchmark, and to ensure that the benchmark finds wide applicability.}, keywords = {BenchFlow, benchmarking}, doi = {10.1145/2668930.2695527}, author = {Marigianna Skouradaki and Vincenzo Ferme and Frank Leymann and Cesare Pautasso and Dieter Roller} } @conference {benchflow:2015:btw, title = {Towards Workflow Benchmarking: Open Research Challenges}, booktitle = {16. Fachtagung Datenbanksysteme f{\"u}r Business, Technologie und Web (BTW)}, year = {2015}, month = {March}, pages = {331-350}, publisher = {Gesellschaft f{\"u}r Informatik}, organization = {Gesellschaft f{\"u}r Informatik}, address = {Hamburg, Germany}, keywords = {BenchFlow, benchmarking, workflow}, url = {http://www.btw-2015.de/?programm_main}, author = {Cesare Pautasso and Dieter Roller and Frank Leymann and Vincenzo Ferme and Marigianna Skouradaki} } @inproceedings {benchflow:zeus2014, title = {"BPELanon": Anonymizing BPEL Processes}, year = {2014}, month = {February}, address = {Potsdam, Germany}, abstract = {We are currently developing a performance benchmark for Workflow Management System. As a first activity we are collecting real-world processes. However, to protect their competitive advantage, some companies are not willing to share their corporate assets. This work{\textquoteright}s objective is to propose a method ("BPELanon") for BPEL process anonymization in order to deal with the problem. The method transforms a process to preserve its original structure and runtime behaviour, while completely anonymizing its business semantics. Anonymization is a complicated task that must meet the requirements we outline in this paper. Namely, we need to preserve the structural and executional information while anonymizing information such as namespaces, names (activity names, variable names, partner link names etc.), and XPATH expressions that may reveal proprietary information. Furthermore, the names contained in the anonymized process should be chosen carefully in order to avoid conflicts, preserve privacy, and file-readability. Multiple dependency relations among process artifacts raise the challenge of fulfilling the aforementioned requirements, as a unique change in a file potentially leads to a flow of changes to other related process artifacts. }, keywords = {Anonymization, BenchFlow, BPEL, business process modeling}, author = {Marigianna Skouradaki and Dieter Roller and Cesare Pautasso and Frank Leymann} } @conference {benchflow:2014:sosp, title = {Technical Open Challenges on Benchmarking Workflow Management Systems}, booktitle = {Symposium on Software Performance}, year = {2014}, month = {November}, pages = {105-112}, address = {Stuttgart, Germany}, abstract = {The goal of the BenchFlow project is to design the first benchmark for assessing and comparing the performance of BPMN 2.0 Workflow Management Systems (WfMSs). WfMSs have become the platform to build composite service-oriented applications, whose performance depends on two factors: the performance of the workflow system itself and the performance of the composed services (which could lie outside of the control of the workflow). Our main goal is to present to the community the state of our work, and the open challenges of a complex industry-relevant benchmark}, keywords = {BenchFlow, benchmarking, workflow engine}, url = {http://www.performance-symposium.org/2014/proceedings/}, author = {Marigianna Skouradaki and Dieter Roller and Frank Leymann and Vincenzo Ferme and Cesare Pautasso} }