@conference {2017:europlop,
	title = {A Pattern Language for Workflow Engine Conformance and Performance Benchmarking},
	booktitle = {22nd European Conference on Pattern Languages of Programs (EuroPLoP)},
	year = {2017},
	month = {July},
	publisher = {ACM},
	organization = {ACM},
	address = {Kloster Irsee, Germany},
	abstract = {Workflow engines are frequently used in the domains of business process management, service orchestration, and cloud computing, where they serve as middleware platforms for integrated business applications. Engines have a significant impact on the quality of service provided by hosted applications. Therefore, it is desirable to compare them and to select the most appropriate engine for a given task. To enable such a comparison, approaches for benchmarking workflow engines have emerged. Although these approaches deal with different quality attributes, i.e., performance or standard conformance, they face many reoccurring design and implementation problems, which have been solved in similar ways. In this paper, we present a pattern language that captures such common solutions to reoccurring problems (e.g., from test identification, benchmarking procedure validation, automatic engine interaction, and workflow execution observation) in the area of workflow engine conformance and performance benchmarking. Our aim is to help future benchmark authors with the pattern language presented in this paper to benefit from our experience with the design and implementation of workflow engine benchmarks and benchmarking tools},
	keywords = {BenchFlow, pattern language},
	doi = {10.1145/3147704.3147705},
	author = {Simon Harrer and J{\"o}rg Lenhard and Oliver Kopp and Vincenzo Ferme and Cesare Pautasso}
}
@proceedings {2017:benchflow:bpmds,
	title = {Performance Comparison Between BPMN 2.0 Workflow Management Systems Versions},
	year = {2017},
	month = {June},
	publisher = {Springer},
	address = {Essen, Germany},
	abstract = {Software has become a rapidly evolving artifact and Workflow Management Systems (WfMSs) are not an exception. WfMSs{\textquoteright} changes may impact key performance indicators or resource consumption levels may change among different versions. Thus, users considering a WfMS upgrade need to evaluate the extent of such changes for frequently issued workload.  Deriving such information requires running performance experiments with appropriate workloads. 
In this paper, we propose a novel method for deriving a structurally representative workload from a given business process collection, which we later use to evaluate the performance and resource consumption over four versions of two open-source WfMSs, for different numbers of simulated users. In our case study scenario the results reveal relevant variations in the WfMSs{\textquoteright} performance and resource consumption, indicating a decrease in performance for newer versions.},
	keywords = {BenchFlow, BPMN, Performance Regression, Performance Testing, workflow engine, Workflow Management Systems},
	author = {Vincenzo Ferme and Marigianna Skouradaki and Ana Ivanchikj and Cesare Pautasso and Frank Leymann}
}
@conference {benchflow:2017:bpm,
	title = {On the Performance Overhead of BPMN Modeling Practices},
	booktitle = {15th International Conference on Business Process Management (BPM2017)},
	year = {2017},
	month = {September},
	pages = {216--232},
	publisher = {Springer},
	organization = {Springer},
	address = {Barcelona, Spain},
	abstract = {Business process models can serve different purposes, from discussion and analysis among stakeholders, to simulation and execution. 
While work has been done on deriving modeling guidelines to improve understandability, it remains to be determined how different modeling practices impact the execution of the models. 
In this paper we observe how semantically equivalent, but syntactically different, models behave in order to assess the performance impact of different modeling practices. 
To do so, we propose a methodology for systematically deriving semantically equivalent models by applying a set of model transformation rules and for precisely measuring their execution performance. 
We apply the methodology on three scenarios to systematically explore the performance variability of 16 different versions of parallel, exclusive, and inclusive control flows. 
Our experiments with two open-source business process management systems measure the execution duration of each model{\textquoteright}s instances. 
The results reveal statistically different execution performance when applying different modeling practices without total ordering of performance ranks.
},
	keywords = {BenchFlow, BPMN, performance},
	doi = {10.1007/978-3-319-65000-5_13},
	author = {Ana Ivanchikj and Vincenzo Ferme and Cesare Pautasso}
}
