@proceedings {2017:benchflow:bpmds, title = {Performance Comparison Between BPMN 2.0 Workflow Management Systems Versions}, year = {2017}, month = {June}, publisher = {Springer}, address = {Essen, Germany}, abstract = {Software has become a rapidly evolving artifact and Workflow Management Systems (WfMSs) are not an exception. WfMSs{\textquoteright} changes may impact key performance indicators or resource consumption levels may change among different versions. Thus, users considering a WfMS upgrade need to evaluate the extent of such changes for frequently issued workload. Deriving such information requires running performance experiments with appropriate workloads. In this paper, we propose a novel method for deriving a structurally representative workload from a given business process collection, which we later use to evaluate the performance and resource consumption over four versions of two open-source WfMSs, for different numbers of simulated users. In our case study scenario the results reveal relevant variations in the WfMSs{\textquoteright} performance and resource consumption, indicating a decrease in performance for newer versions.}, keywords = {BenchFlow, BPMN, Performance Regression, Performance Testing, workflow engine, Workflow Management Systems}, author = {Vincenzo Ferme and Marigianna Skouradaki and Ana Ivanchikj and Cesare Pautasso and Frank Leymann} }