From 4492c5cfbe0e99666afed6780b674beaa670d1a1 Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Fri, 23 Nov 2018 09:27:06 -0800 Subject: [bash] Let's try to use emacs tty --- playgrounds/aurora/fio.aurora | 45 ++++++++++++++++++++ playgrounds/aurora/hello-job.aurora | 22 ++++++++++ playgrounds/aurora/iperf.aurora | 37 +++++++++++++++++ playgrounds/aurora/smf1-test-cron-job.aurora | 17 ++++++++ playgrounds/fio-bench.tgz | Bin 0 -> 1189 bytes playgrounds/fio/rand-read.fio | 16 +++++++ playgrounds/fio/rand-write.fio | 14 +++++++ playgrounds/fio/runner.sh | 60 +++++++++++++++++++++++++++ playgrounds/fio/seq-read.fio | 14 +++++++ playgrounds/fio/seq-rw.fio | 17 ++++++++ playgrounds/fio/seq-write.fio | 14 +++++++ 11 files changed, 256 insertions(+) create mode 100644 playgrounds/aurora/fio.aurora create mode 100644 playgrounds/aurora/hello-job.aurora create mode 100644 playgrounds/aurora/iperf.aurora create mode 100644 playgrounds/aurora/smf1-test-cron-job.aurora create mode 100644 playgrounds/fio-bench.tgz create mode 100644 playgrounds/fio/rand-read.fio create mode 100644 playgrounds/fio/rand-write.fio create mode 100755 playgrounds/fio/runner.sh create mode 100644 playgrounds/fio/seq-read.fio create mode 100644 playgrounds/fio/seq-rw.fio create mode 100644 playgrounds/fio/seq-write.fio (limited to 'playgrounds') diff --git a/playgrounds/aurora/fio.aurora b/playgrounds/aurora/fio.aurora new file mode 100644 index 0000000..b26debf --- /dev/null +++ b/playgrounds/aurora/fio.aurora @@ -0,0 +1,45 @@ +downloadFIO = Process( + name='download-fio', + cmdline='curl -o fio.rpm https://svn.twitter.biz/rpms/fio.x86_64/RPMS/x86_64/fio-1.50-2.twitter.x86_64.rpm' +) + +installBenchs = Packer.install( + 'fio-bench', + role = 'fcuny', + version = 'latest' +) + +mvFIO = Process( + name='move-fio', + cmdline='mv fio/* . && rm -rf fio', +) + +extractFIO = Process( + name='extract-fio', + cmdline='rpm2cpio fio.rpm | cpio -idmv' +) + +runFIO = Process( + name='run-fio', + cmdline='./runner.sh', +) + +jobs = [ + Job( + cluster='smf1', + role='fcuny', + environment='devel', + name='fio', + task=Task( + processes=[downloadFIO, extractFIO, runFIO, installBenchs, mvFIO], + resources=Resources(cpu=4, ram=4096 * MB, disk=10 * GB), + constraints=order(installBenchs, mvFIO, downloadFIO, extractFIO, runFIO) + ), + instances=3, + constraints={ + 'base_platform': 'f4ww', + 'dedicated': '*/manhattan', + 'host': 'limit:1', + } + ) +] diff --git a/playgrounds/aurora/hello-job.aurora b/playgrounds/aurora/hello-job.aurora new file mode 100644 index 0000000..a825fdb --- /dev/null +++ b/playgrounds/aurora/hello-job.aurora @@ -0,0 +1,22 @@ +hello_date = Process( + name='hello-date', + cmdline='while true; do date; sleep 10; done' +) + +jobs = [ + Service( + cluster='smf1', + environment='devel', + role='fcuny', + name='hello-date', + task=Task( + processes=[hello_date], + resources=Resources(cpu=10, ram=1024 * MB, disk=512 * MB) + ), + instances=1, + constraints={ + 'host': 'smf1-fki-16-sr1', + 'dedicated': '*/manhattan', + } + ) +] diff --git a/playgrounds/aurora/iperf.aurora b/playgrounds/aurora/iperf.aurora new file mode 100644 index 0000000..e071bd2 --- /dev/null +++ b/playgrounds/aurora/iperf.aurora @@ -0,0 +1,37 @@ +class StandardProfile(Struct): + environment=Default(String, 'prod') + tier=Default(String, 'preferred') + +DevelProfile = StandardProfile( + environment = 'devel', + tier = 'preemptible', +) + +api = Process( + name = 'iperf', + cmdline = '/usr/bin/iperf3 -s -p {{thermos.ports[http]}}' +) + +task = Task( + name = api.name(), + resources = Resources(cpu = 1.0, ram = 4 * GB, disk = 1 * GB), + processes = [api] +) + +service_template = Service( + role='fcuny', + name = 'iperf', + environment='{{profile.environment}}', + task = task, + instances = 3, + contact = 'fcuny@twitter.com', + announce=Announcer(), + tier ='{{profile.tier}}', + constraints={ + 'host': 'smf1-bgr-27-sr1', + } +) + +jobs = [ + service_template(cluster='smf1-test').bind(profile=DevelProfile()), +] diff --git a/playgrounds/aurora/smf1-test-cron-job.aurora b/playgrounds/aurora/smf1-test-cron-job.aurora new file mode 100644 index 0000000..3ade1b4 --- /dev/null +++ b/playgrounds/aurora/smf1-test-cron-job.aurora @@ -0,0 +1,17 @@ +# A cron job that runs every 5 minutes. +jobs = [ + Job( + cluster = 'smf1-test', + role = 'fcuny', + environment = 'test', + name = 'cron_hello_world-trashing', + cron_schedule = '*/5 * * * *', + constraints = { + 'host': 'smf1-fki-17-sr1', + }, + instances=10, + task = SimpleTask( + 'cron_hello_world', + 'echo "Hello world from cron, the time is now $(date --rfc-822)"'), + ), +] diff --git a/playgrounds/fio-bench.tgz b/playgrounds/fio-bench.tgz new file mode 100644 index 0000000..06ff644 Binary files /dev/null and b/playgrounds/fio-bench.tgz differ diff --git a/playgrounds/fio/rand-read.fio b/playgrounds/fio/rand-read.fio new file mode 100644 index 0000000..8d473af --- /dev/null +++ b/playgrounds/fio/rand-read.fio @@ -0,0 +1,16 @@ +[global] +name=fio-rand-RW +filename=./data/fio-rand-RW +rw=randrw +rwmixread=60 +rwmixwrite=40 +bs=4K +direct=0 +numjobs=4 +time_based=1 +runtime=900 + +[file1] +size=8G +ioengine=libaio +iodepth=16 diff --git a/playgrounds/fio/rand-write.fio b/playgrounds/fio/rand-write.fio new file mode 100644 index 0000000..5564414 --- /dev/null +++ b/playgrounds/fio/rand-write.fio @@ -0,0 +1,14 @@ +[global] +name=fio-rand-write +filename=./data/fio-rand-write +rw=randwrite +bs=4K +direct=0 +numjobs=4 +time_based=1 +runtime=900 + +[file1] +size=8G +ioengine=libaio +iodepth=16 diff --git a/playgrounds/fio/runner.sh b/playgrounds/fio/runner.sh new file mode 100755 index 0000000..8df8895 --- /dev/null +++ b/playgrounds/fio/runner.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +## Helper script to run fio tests and generate reports + +DATA_DIR=./data +LOG_DIR=./logs +REPORT_DIR=./reports + +FIO_BIN=./usr/bin/fio +if [ "${2}x" != "x" ]; then + FIO_BIN=${2} +fi + +FIOS_LIST=$(ls *.fio) +NOW_EPOCH=$(date +"%s") +LOG_DIR_READS=${LOG_DIR}/reads +LOG_DIR_WRITES=${LOG_DIR}/writes + +# create required directories +mkdir -p ${DATA_DIR} + +if [ -d "${REPORT_DIR}" ]; then + echo "Report directory exists, archiving using current timestamp: ${NOW_EPOCH}" + mv ${REPORT_DIR} ${REPORT_DIR}_${NOW_EPOCH} +fi +mkdir -p ${REPORT_DIR} + +if [ -d "${LOG_DIR}" ]; then + echo "Log directory exists, archiving using current timestamp: ${NOW_EPOCH}" + mv ${LOG_DIR} ${LOG_DIR}_${NOW_EPOCH} +fi +mkdir -p ${LOG_DIR_WRITES} +mkdir -p ${LOG_DIR_READS} + +# run all fios in sequential order +for i in $(echo ${FIOS_LIST} | tr " " "\n") +do + echo -e "\nStarting fio test ${i}..." + ${FIO_BIN} ./${i} --output ${REPORT_DIR}/${i}.out + + mv *read*.log ${LOG_DIR_READS}/ + mv *write*.log ${LOG_DIR_WRITES}/ + + rm -f data/* # delete created fio files after each run + + echo "Completed fio test ${i}." +done + +# plot reports to svg +FIO_PLOT_BIN=./usr/bin/fio_generate_plots +if type "${FIO_PLOT_BIN}" > /dev/null && type "gnuplot" > /dev/null; then + + echo "fio_generate_plots is installed generating svg reports based on fio logs" + + ( cd ${LOG_DIR_READS} && for f in *.log; do mv $f ${f/.[1-3]/}; done && ${FIO_PLOT_BIN} "All-Reads" ) + ( cd ${LOG_DIR_WRITES} && for f in *.log; do mv $f ${f/.[1-3]/}; done && ${FIO_PLOT_BIN} "All-Writes" ) + + mv ${LOG_DIR_READS}/*.svg ${REPORT_DIR}/ + mv ${LOG_DIR_WRITES}/*.svg ${REPORT_DIR}/ +fi diff --git a/playgrounds/fio/seq-read.fio b/playgrounds/fio/seq-read.fio new file mode 100644 index 0000000..c3225a3 --- /dev/null +++ b/playgrounds/fio/seq-read.fio @@ -0,0 +1,14 @@ +[global] +name=fio-seq-reads +filename=./data/fio-seq-reads +rw=read +bs=256K +direct=1 +numjobs=1 +time_based=1 +runtime=900 + +[file1] +size=8G +ioengine=libaio +iodepth=16 diff --git a/playgrounds/fio/seq-rw.fio b/playgrounds/fio/seq-rw.fio new file mode 100644 index 0000000..81c50e7 --- /dev/null +++ b/playgrounds/fio/seq-rw.fio @@ -0,0 +1,17 @@ +[global] +name=fio-seq-RW +filename=./data/fio-seq-RW +rw=rw +rwmixread=60 +rwmixwrite=40 +bs=256K +direct=0 +numjobs=4 +time_based=1 +runtime=900 + +[file1] +size=8G +ioengine=libaio +iodepth=16 + diff --git a/playgrounds/fio/seq-write.fio b/playgrounds/fio/seq-write.fio new file mode 100644 index 0000000..88fbeaa --- /dev/null +++ b/playgrounds/fio/seq-write.fio @@ -0,0 +1,14 @@ +[global] +name=fio-seq-write +filename=./data/fio-seq-write +rw=write +bs=256K +direct=0 +numjobs=1 +time_based=1 +runtime=900 + +[file1] +size=8G +ioengine=libaio +iodepth=16 -- cgit 1.4.1