@conference {2018:benchflow:coopis, title = {Evaluating Multi-Tenant Live Migrations Effects on Performance}, booktitle = {26th International Conference on Cooperative Information Systems (CoopIS)}, year = {2018}, month = {October}, address = {Valletta, Malta}, abstract = {Multitenancy is an important feature for all Everything as a Service providers like Business Process Management as a Service. It allows to reduce the cost of the infrastructure since multiple tenants share the same service instances. However, tenants have dynamic workloads. The resource they share may not be sufficient at some point in time. It may require Cloud resource (re-)configurations to ensure a given Quality of Service. Tenants should be migrated without stopping the service from a configuration to another to meet their needs while minimizing operational costs on the provider side. Live migrations reveal many challenges: service interruption must be minimized and the impact on co-tenants should be minimal. In this paper, we investigate live tenants migrations duration and its effects on the migrated tenants as well as the co-located ones. To do so, we propose a generic approach to measure these effects for multi-tenant Software as a Service. Further, we propose a testing framework to simulate workloads, and observe the impact of live migrations on Business Process Management Systems. The experimental results highlight the efficiency of our approach and show that migration time depends on the size of data that have to be transferred and that the effects on co-located tenants should not be neglected. }, keywords = {BenchFlow, Multi-tenant, Performance Testing, workflow engine}, author = {Guillaume Rosinosky and Chahrazed Labba and Vincenzo Ferme and Samir Youcef and Fran{\c c}ois Charoy and Cesare Pautasso} } @proceedings {2017:benchflow:bpmds, title = {Performance Comparison Between BPMN 2.0 Workflow Management Systems Versions}, year = {2017}, month = {June}, publisher = {Springer}, address = {Essen, Germany}, abstract = {Software has become a rapidly evolving artifact and Workflow Management Systems (WfMSs) are not an exception. WfMSs{\textquoteright} changes may impact key performance indicators or resource consumption levels may change among different versions. Thus, users considering a WfMS upgrade need to evaluate the extent of such changes for frequently issued workload. Deriving such information requires running performance experiments with appropriate workloads. In this paper, we propose a novel method for deriving a structurally representative workload from a given business process collection, which we later use to evaluate the performance and resource consumption over four versions of two open-source WfMSs, for different numbers of simulated users. In our case study scenario the results reveal relevant variations in the WfMSs{\textquoteright} performance and resource consumption, indicating a decrease in performance for newer versions.}, keywords = {BenchFlow, BPMN, Performance Regression, Performance Testing, workflow engine, Workflow Management Systems}, author = {Vincenzo Ferme and Marigianna Skouradaki and Ana Ivanchikj and Cesare Pautasso and Frank Leymann} } @conference {benchflow:2016:caise, title = {Micro-Benchmarking BPMN 2.0 Workflow Management Systems with Workflow Patterns}, booktitle = {Proc. of the 28th International Conference on Advanced Information Systems Engineering (CAISE)}, year = {2016}, month = {June}, pages = {67--82}, publisher = {Springer}, organization = {Springer}, address = {Ljubljana, Slovenia}, abstract = {Although Workflow Management Systems (WfMSs) are a key component in workflow technology, research work for assessing and comparing their performance is limited. This work proposes the first micro- benchmark for WfMSs that can execute BPMN 2.0 workflows. To this end, we focus on studying the performance impact of well-known workflow patterns expressed in BPMN 2.0 with respect to three open source WfMSs (i.e., Activiti, jBPM and Camunda). We executed all the experiments under a reliable environment and produced a set of meaningful metrics. This paper contributes to the area of workflow technology by defining building blocks for more complex BPMN 2.0 WfMS benchmarks. The results have shown bottlenecks on architectural design decisions, resource utilization, and limits on the load a WfMS can sustain, especially for the cases of complex and parallel structures. Experiments on a mix of workflow patterns indicated that there are no unexpected performance side effects when executing different workflow patterns concurrently, although the duration of the individual workflows that comprised the mix was increased.}, keywords = {BenchFlow, benchmarking, BPMN, Microbenchmark, workflow engine, Workflow Management Systems, workflow patterns}, doi = {10.1007/978-3-319-39696-5_5}, url = {http://www2.informatik.uni-stuttgart.de/cgi-bin/NCSTRL/NCSTRL_view.pl?id=INPROC-2016-05\&engl=0}, author = {Marigianna Skouradaki and Vincenzo Ferme and Cesare Pautasso and Frank Leymann and Andr{\'e} van Hoorn} } @conference {benchflow:2014:sosp, title = {Technical Open Challenges on Benchmarking Workflow Management Systems}, booktitle = {Symposium on Software Performance}, year = {2014}, month = {November}, pages = {105-112}, address = {Stuttgart, Germany}, abstract = {The goal of the BenchFlow project is to design the first benchmark for assessing and comparing the performance of BPMN 2.0 Workflow Management Systems (WfMSs). WfMSs have become the platform to build composite service-oriented applications, whose performance depends on two factors: the performance of the workflow system itself and the performance of the composed services (which could lie outside of the control of the workflow). Our main goal is to present to the community the state of our work, and the open challenges of a complex industry-relevant benchmark}, keywords = {BenchFlow, benchmarking, workflow engine}, url = {http://www.performance-symposium.org/2014/proceedings/}, author = {Marigianna Skouradaki and Dieter Roller and Frank Leymann and Vincenzo Ferme and Cesare Pautasso} }