@conference {benchflow:2016:closer, title = {A Container-centric Methodology for Benchmarking Workflow Management Systems}, booktitle = {6th International Conference on Cloud Computing and Service Science (CLOSER 2016)}, year = {2016}, month = {April}, pages = {74-84}, publisher = {SciTePress}, organization = {SciTePress}, address = {Rome, Italy}, abstract = {Trusted benchmarks should provide reproducible results obtained following a transparent and well-defined process. In this paper, we show how Containers, originally developed to ease the automated deployment of Cloud application components, can be used in the context of a benchmarking methodology. The proposed methodology focuses on Workflow Management Systems (WfMSs), a critical service orchestration middleware, which can be characterized by its architectural complexity, for which Docker Containers offer a highly suitable approach. The contributions of our work are: 1) a new benchmarking approach taking full advantage of containerization technologies; and 2) the formalization of the interaction process with the WfMS vendors described clearly in a written agreement. Thus, we take advantage of emerging Cloud technologies to address technical challenges, ensuring the performance measurements can be trusted. We also make the benchmarking process transparent, automated, and repeatable so that WfMS vendors can join the benchmarking effort.}, keywords = {BenchFlow, benchmarking, Docker}, doi = {10.5220/0005908400740084}, author = {Vincenzo Ferme and Ana Ivanchikj and Cesare Pautasso and Marigianna Skouradaki and Frank Leymann} } @demo {benchflow:2016:icpe, title = {Integrating Faban with Docker for Performance Benchmarking}, year = {2016}, month = {March}, pages = {129-130}, publisher = {ACM}, address = {Delft, The Netherlands}, abstract = {Reliability and repeatability are key requirements in performance benchmarking ensuring the trustworthiness of the obtained performance results. To apply a benchmark to multiple systems, the reusability of the load driver is essential. While Faban has been designed to ensure the reliability of the performance data obtained from a benchmark experiment, it lacks support for ensuring that the system under test is deployed in a known configuration. This is what Docker, a recently emerging containerization technology, excels at. In this demo paper we present how we integrated Faban with Docker as part of the BenchFlow framework to offer a complete and automated performance benchmarking framework that provides a reliable and reusable environment, ensuring the repeatability of the experiments. }, keywords = {BenchFlow, benchmarking, Docker, Faban}, doi = {10.1145/2851553.2858676}, author = {Vincenzo Ferme and Cesare Pautasso} }