Misc code for distributing oltpcore/mtitcore processes

Author: bengsig

bin/mtitcore

@@ -10,7 +10,8 @@
 
 # Changes
 #
-# bengsig  10-may-2023 Creation
+# bengsig   9-nov-2023 - support multiple application servers
+# bengsig  10-may-2023 - Creation
 
 # If stdin is not a tty 
 # show how we were called
@@ -156,7 +157,7 @@ fi
 eval `rwloadsim -q -i proccount:=$procs -i runperiod:=$runperiod parameters2shell.rwl`
 
 # verify that it compiles
-rwloadsim -u -v $extra_args -e -- '-x $mute:141' $runfile || exit $?
+rwloadsim -u -v $extra_args -e '-x $mute:141' $runfile || exit $?
 
 if test $mtitxc = yes
 then
@@ -189,7 +190,7 @@ then
   rwloadsim -u -v -c 15 -sss -P $prepfile -k $key -K "$komment" -q \
   --fulltitle="$komment" \
   -i runperiod:=$runperiod -i proccount:=$procs prepare.rwl
-  mute59='-x$mute:59'
+  mute59='--mute=59'
 else
   rwloadsim -u -v -c $rampup -sss -P $prepfile -k $key -K "$komment" -q \
   --fulltitle="$komment" \
@@ -227,7 +228,6 @@ then
   echo $! >> $killfile
 fi
 
-markerfile='string(1000)xc_marker_file:="'$resultsdir/$subdir/$runnumber/markers.txt'";'
 quiet=''
 
 # Start sys gather
@@ -239,17 +239,20 @@ then
   echo $awrpid >> $killfile
 fi
 
+prepvalue=`cat $prepfile`
 # show countdown to user
 echo -n '**** remaining:'
 # Start the real workers
 while test $procnumber -le $procs
 do
   sleep $script_ramp
   echo -n " "`expr $procs - $procnumber`
-  rwloadsim $mute59 -u -r -i procnumber:=$procnumber \
+  eval $(rwloadsim -q -i proccount:=$procs -i runperiod:=$runperiod rwlcall.rwl -i procnumber:=$procnumber)
+
+  $rwlcall $rwlcallargs $mute59 -u -r -i procnumber:=$procnumber \
     --flush-stop=$runperiod --flush-every=2 -v -i runperiod:=$runperiod \
-    --procno $procnumber \
-    -W -i doawr:=$doawr -x "$markerfile" $mtitsetxc $extra_args -i proccount:=$procs -R $prepfile $quiet $runfile &
+    --procno $procnumber --markerfile=$resultsdir/$subdir/$runnumber/markers.txt \
+    -W -i doawr:=$doawr $mtitsetxc $extra_args -i proccount:=$procs -M $prepvalue $quiet $runfile &
   quiet='-q' # only messages from first
   procnumber=`expr $procnumber + 1`
   if test x$killfile != x

bin/oltpcore

@@ -11,6 +11,7 @@
 
 # Changes
 #
+# bengsig   9-nov-2023 - support multiple application servers
 # bengsig   2-nov-2023 - Add extra_args plots, etc
 # bengsig  17-oct-2023 - Add extra_args to prepare
 # bengsig  29-sep-2023 - No ash if no awr
@@ -179,7 +180,7 @@ fi
 eval `rwloadsim -q $extra_args -i proccount:=$procs -i runperiod:=$runperiod parameters2shell.rwl`
 
 # verify that it compiles
-rwloadsim -q -u -v $extra_args -e -- '-x $mute:141' $runfile || exit $?
+rwloadsim -q -u -v $extra_args -e '-x $mute:141' $runfile || exit $?
 
 if test $oltpxc = yes
 then
@@ -215,7 +216,7 @@ then
   rwloadsim $extra_args -u -v -c 15 -sss -P $prepfile -k $key -K "$komment" -q \
   --fulltitle="$komment" \
   -i runperiod:=$runperiod -i proccount:=$procs prepare.rwl
-  mute59='-x$mute:59'
+  mute59='--mute=59'
 else
   rwloadsim $extra_args -u -v -c $rampup -sss -P $prepfile -k $key -K "$komment" -q \
   --fulltitle="$komment" \
@@ -255,7 +256,6 @@ then
   echo $! >> $killfile
 fi
 
-markerfile='string(1000)xc_marker_file:="'$resultsdir/$subdir/$runnumber/markers.txt'";'
 quiet=''
 
 # Start sys gather
@@ -273,16 +273,21 @@ then
   rwloadsim -u -q -i simulatebatch:=$simulatebatch -i runperiod:=$runperiod -R $prepfile runtrunc.rwl &
 fi
 
+prepvalue=`cat $prepfile`
+
 # show countdown to user
 echo -n '**** remaining:'
 # Start the real workers
 while test $procnumber -le $procs
 do
   sleep $script_ramp
   echo -n " "`expr $procs - $procnumber`
-  rwloadsim $mute59 -u -r -i simulatebatch:=$simulatebatch -i procnumber:=$procnumber \
+  # get the actual call to execute in background
+  eval $(rwloadsim -q -i proccount:=$procs -i runperiod:=$runperiod rwlcall.rwl -i procnumber:=$procnumber)
+  # by default, rwlcall=rwloadsim and rwlcallargs is empty
+  $rwlcall $rwlcallargs $mute59 -u -r -i simulatebatch:=$simulatebatch -i procnumber:=$procnumber \
     --flush-stop=$runperiod --flush-every=2 -v -i runperiod:=$runperiod \
-    -W -i xc_run_dedicated:=$xc_run_dedicated -i doawr:=$doawr -x $markerfile $oltpsetxc $extra_args -i proccount:=$procs -R $prepfile $quiet $runfile &
+    -W -i xc_run_dedicated:=$xc_run_dedicated -i doawr:=$doawr --markerfile=$resultsdir/$subdir/$runnumber/markers.txt $oltpsetxc $extra_args -i proccount:=$procs -M $prepvalue $quiet $runfile &
   quiet='-q' # only messages from first
   procnumber=`expr $procnumber + 1`
   if test x$killfile != x

man/man2rwl/oltpsetup.2rwl

@@ -575,6 +575,43 @@ To verify you can connect to a repository, run \fBoltpverify -r\fR.
 To create all schemas, run \fBoltpcreate\fR; to drop all schemas expect the repository, run \fBoltpdrop\fR.
 .P
 To verify schemas are properly created, run \fBoltpverify -u\fR.
+.SH "REMOTE EXECUTION"
+Normally, all the background processes started by e.g. \fBoltprun\fR are started on the same single host.
+If that host is running out of CPU resources, you can prepare distribution of the processes to
+multiple servers.
+This is done by declaring a procedure named \fBrwlcall\fR taking a single integer argument.
+The argument provided will be a process number in the range from 2 until the value of the -n argument
+to \fBoltprun\fR.
+The process with process number 1 will always be started on the same host where \fBoltprun\fR is executed.
+The procedure must output exactly two lines that are setting sh variables named \fBrwlcall\fR 
+and \fBrwlcallargs\fR.
+Those two will be used to start the actual background process.
+A sample declaration that will distribute the workload to three different hosts is the following:
+.P
+.nf
+procedure rwlcall(integer prno)
+is
+  printline "rwlcall='ssh'";
+  printf "rwlcallargs='-n -tt appsrv%02d.domain.tld /path/to/remoterwloadsim.bash'\\n"  
+    , prno%3+1;
+end rwlcall;
+.fi
+.P
+A template for the bash script \fBremoterwloadsim.bash\fR is found in the oltp directory of your
+rwloadsim distribution.
+It should be copied to all remote systems and modifed according to the instructions in
+the file, setting the proper environment to execute rwloadsim remotely.
+Note that remote execution is an experimental feature that may change in a later release.
+.P
+If the procedure is not declared, the following will be used
+.P
+.nf
+  printline "rwlcall='rwloadsim'";
+  printline "rwlcallargs=''";
+.fi
+.P
+implying rwloadsim will be started locally.
+This is always used for process number 1, as it must run locally.
 .SH "UNDOCUMENTED PARAMETERS"
 There are some extra parameters, that you can set in either your \fBkey.rwl\fR file or using -i or -d on the
 oltprun command line.

oltp/mtitrun.rwl

@@ -2,6 +2,7 @@
 # Licensed under the Universal Permissive License v 1.0
 # as shown at https://oss.oracle.com/licenses/upl/
 
+# bengsig   9-nov-2023 - support multiple application servers
 # bengsig  17-jul-2023 - Call mtit_initialize in each thread
 # bengsig  23-apr-2023 - Creation
 
@@ -18,6 +19,8 @@ $include:"ovid2.rwl"
 $include:"mtit_user.rwl"
 $include:"mtit_declarations.rwl"
 
+string(1000) markerfile; $useroption:markerfile
+# note that markerfile is only used by xc_implementation
 $if mtit_xc $then
  $include:"xc_implementation.rwl"
 

oltp/parameters.rwl

@@ -2,6 +2,8 @@
 # Licensed under the Universal Permissive License v 1.0
 # as shown at https://oss.oracle.com/licenses/upl/
 
+# bengsig   9-nov-2023 - support multiple application servers
+# bengsig   6-nov-2023 - burst function
 # bengsig   1-nov-2023 - burst can be repeated
 # bengsig  17-oct-2023 - output resolution
 # bengsig   3-oct-2023 - parsefailure
@@ -671,6 +673,10 @@ if show_changed_values then
   end if;
   printline "aw_cols_256MB_doublings=" aw_cols_256MB_doublings;
   printline "aw_cols_skew=" aw_cols_skew;
+  $if defined(rwlcall) $then
+    printline "processes will be running via user defined ssh with this as an example:";
+    rwlcall(2);
+  $endif
 end if;
 
 string svgmouse ;

oltp/remoterwloadsim.bash

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# RWP*Load Simulator - Call rwloadsime remotely
+#
+# Copyright (c) 2021 Oracle Corporation
+# Licensed under the Universal Permissive License v 1.0
+# as shown at https://oss.oracle.com/licenses/upl/
+#
+# History
+#
+# bengsig   8-nov-2023 - Creation
+#
+# This file is a sample of how to implement spreading of 
+# oltp worker processes remotely. It will be called using 
+# ssh with an unknown environment that can depend on sevaral
+# factors, so ALL environment variables below must be set
+# properly
+
+# You need to copy this file to all your remote application
+# servers and provide the full name to it in your implementation
+# of the rwlcall function
+#
+# Note that things will be much easier if the home directory
+# of the account you are using and the rwloadsim distribution
+# are shared (e.g. via NFS) to all application servers.
+
+# Set the RWLOADSIM_PATH such that it includes the following two
+# directories on the remote system:
+# the directory where a copy of {key}.rwl is found
+# the oltp directory of your rwloadsim distribution
+# Effectively, this is similar to your normal {key}.env file
+export RWLOADSIM_PATH=/path/to/keydirectory:/path/to/rwloadsim/oltp
+export RWLOLTP_NAME=key # put your project name here
+# Set the following two such that it provides access to rwloadsim, sqlplus, etc:
+export LD_LIBRARY_PATH=/path/to/instantclient
+PATH=/path/to/rwloadsim/bin:/bin:/usr/bin:/usr/local/bin:/path/to/instantclient
+# Change to the directory where {key}.rwl is found
+cd /path/to/keydirectory
+
+# finally do the actual call to rwloadsim
+rwloadsim "$@"
+

oltp/run.rwl

@@ -2,6 +2,7 @@
 # Licensed under the Universal Permissive License v 1.0
 # as shown at https://oss.oracle.com/licenses/upl/
 
+# bengsig   6-nov-2023 - burst function
 # bengsig   1-nov-2023 - burst can be repeated
 # bengsig  11-sep-2023 - max_sessionwait
 # bengsig  31-jul-2023 - Add experimental 24h simulation
@@ -16,6 +17,7 @@
 
 integer xc_run_dedicated := 0; 
 
+
 $statistics:all
 
 $include:"parameters.rwl"
@@ -34,6 +36,8 @@ $include:"oe_handle_products.rwl"
 $include:"aw_declarations.rwl"
 $include:"oe_long_session.rwl"
 
+string(1000) markerfile; $useroption:markerfile
+# note that markerfile is only used by xc_implementation
 $if xc_enabled $then
  $include:"xc_implementation.rwl"
 $endif

oltp/rwlcall.rwl

@@ -0,0 +1,39 @@
+# Copyright (c) 2021 Oracle Corporation
+# Licensed under the Universal Permissive License v 1.0
+# as shown at https://oss.oracle.com/licenses/upl/
+
+# bengsig   7-nov-2023 - Creation
+
+#
+# If the rwlcall function exists, call it with the process number
+# as argument, unless process number is 1. Process number 1
+# must always executed on the host where you run oltpcore.
+#
+# The output from this will be used in eval $(rwloadsim rwlcall.rwl)
+# in oltpcore (and mtitcore) and it should therefore output
+# nothing more than sh commands to set the two variables rwlcall
+# and rwlcallargs.
+#
+# The default is to print this:
+#
+# rwlcall=rwloadsim
+# rwlcallargs=''
+#
+# which will imply the real worker threads will be executed on the host
+# calling oltpcore simply using rwloadsim
+#
+# The function provided by the user is supposed to print something like:
+# rwlcall=ssh
+# rwlcallargs='-n -tt hostname /path/to/somewhere/remoterwloadsim.bash'
+# where the bash script is a modified version of the one found in the oltp
+# directory of your rwloadsim distribution.
+#
+$include:"parameters.rwl"
+
+$if defined(rwlcall) and procnumber > 1 $then
+  rwlcall(procnumber);
+$else
+  printline "rwlcall='rwloadsim'";
+  printline "rwlcallargs=''";
+$endif
+  

oltp/xc_implementation.rwl

@@ -7,6 +7,7 @@
 #
 # History
 #
+#  bengsig   9-nov-2023 - support multiple application servers
 #  bengsig   3-oct-2023 - parsefailure
 #  bengsig   1-aug-2023 - Add oltpxcset markers from oltpxcrun
 #  bengsig  17-jul-2023 - Show affinity properly
@@ -19,18 +20,26 @@ $niceabort:on
 $queueeverytiming:on
 $mute:206
 
-string(1000) xc_marker_file := "";
+$if !defined(markerfile) $then
+# This is a bit of a hack: We need to have markerfile declared in run.rwl
+# such that it can be associated with a --markerfile useroption, as otherwise
+# oltpcore cannot provide a value to it. It is really only used if xc_read
+# is called with its argument repchg set to 1 (true) in which case the
+# changes picked up by xc_read are writting to stdout and markerfile.
+# And xc_read is only called with repchg true when procnumber=1 in oltpcore
+string(1000) markerfile := "";
+$endif
 
-if xc_marker_file != "" then
-  system "rm -f " xc_marker_file;
+if markerfile != "" then
+  system "rm -f " markerfile;
 end;
 
 procedure xc_marker(string txt)
-  if xc_marker_file = "" then
+  if markerfile = "" then
     return;
   end;
 
-  file mf >>= xc_marker_file;
+  file mf >>= markerfile;
   fprintf mf, "%.1f %s\n", runseconds(), txt;
   mf := null;
 end;

src/rwl.h

@@ -13,6 +13,7 @@
  *
  * History
  *
+ * bengsig   9-nov-2023 - Increase RWL_MAX_VAR
  * bengsig   3-oct-2023 - Now development for 3.1.1
  * bengsig   2-oct-2023 - Releasing 3.1.0 
  * bengsig  22-sep-2023 - ampersand fixes
@@ -1281,7 +1282,7 @@ struct rwl_identifier
     rwlmutexrel(xev,loc,(xev)->rwm->mxq->evar[(vnum)].var_mutex); \
   } while (0)
 
-#define RWL_MAX_VAR 500 /* default number of variables */
+#define RWL_MAX_VAR 600 /* default number of variables */
 // This is an array that we do not increase in runtime
 // Doing so would be major change, but user can specify -I
 // to set a larger size