diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index 24e70491184..7a81781d2b1 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -15,11 +15,11 @@ $(recurse) PROGRAMS= analyzedb gpactivatestandby gpaddmirrors gpcheckcat gpcheckperf \ gpcheckresgroupimpl gpconfig gpdeletesystem gpexpand gpshrink gpinitstandby \ gpinitsystem gpload gpload.py gplogfilter gpmovemirrors \ - gppkg gprecoverseg gpreload gpscp gpsd gpssh gpssh-exkeys gpstart \ + gppkg gprecoverseg gpreload gpsync gpsd gpssh gpssh-exkeys gpstart \ gpstate gpstop minirepro gpmemwatcher gpmemreport gpdemo gpdirtableload -GPDEMO_LIBS = gpdemo-defaults.sh lalshell generate_certs.sh demo_cluster.sh \ - probe_config.sh README + GPDEMO_LIBS = gpdemo-defaults.sh lalshell generate_certs.sh demo_cluster.sh \ + probe_config.sh README installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/lib' @@ -115,7 +115,7 @@ pyyaml: @echo "--- pyyaml" cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYYAML_DIR).tar.gz cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC python3 setup.py build - cp -r $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/build/lib*-3*/* $(PYLIB_DIR) + cp -r $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/build/lib*-3.*/* $(PYLIB_DIR) # # PYLINT @@ -195,6 +195,6 @@ clean distclean: rm -f analyzedbc gpactivatestandbyc gpaddmirrorsc gpcheckcatc \ gpcheckperfc gpcheckresgroupimplc gpchecksubnetcfgc gpconfigc \ gpdeletesystemc gpexpandc gpshrinkc gpinitstandbyc gplogfilterc gpmovemirrorsc \ - gppkgc gprecoversegc gpreloadc gpscpc gpsdc gpssh-exkeysc gpsshc \ + gppkgc gprecoversegc gpreloadc gpscpc gpsyncc gpsdc gpssh-exkeysc gpsshc \ gpstartc gpstatec gpstopc minireproc rm -f gpconfig_modules/gucs_disallowed_in_file.txt diff --git a/gpMgmt/bin/README.md b/gpMgmt/bin/README.md index 674a209e0f0..a1600a9d662 100644 --- a/gpMgmt/bin/README.md +++ b/gpMgmt/bin/README.md @@ -42,7 +42,7 @@ List of Management Scripts Written in Python (gpmlib - old libraries) bin/gpaddmirrors - Adds mirrors to an array (needs rewrite) bin/gprecoverseg - Recovers a failed segment (needs rewrite) bin/gpcheckperf - Checks the hardware for Cloudberry Database -bin/gpscp - Copies files to many hosts +bin/gpsync - Copies files to many hosts bin/gpssh - Remote shell to many hosts bin/gpssh-exkeys - Exchange ssh keys between many hosts @@ -144,8 +144,8 @@ db/dbconn.py - Connections to the database +- Should have a wrapper class around a pygresql connection object! util/gp_utils.py - Cloudberry related utility functions that are not Commands -util/ssh_session.py - SSH and SCP related utility functions brought in from gpmlib.py/gplib.py - that are used by gpssh, gpscp and gpssh-exkeys +util/ssh_session.py - SSH and RSYNC related utility functions brought in from gpmlib.py/gplib.py + that are used by gpssh, gpsync and gpssh-exkeys ## Testing Management Scripts (unit tests) diff --git a/gpMgmt/bin/gpcheckperf b/gpMgmt/bin/gpcheckperf index 79ca3ec2792..c4b61b0f399 100755 --- a/gpMgmt/bin/gpcheckperf +++ b/gpMgmt/bin/gpcheckperf @@ -29,6 +29,7 @@ Usage: gpcheckperf -f file : a file listing all hosts to connect to --duration : how long to run network test (default 5 seconds) --netperf : use netperf instead of gpnetbenchServer/gpnetbenchClient + --buffer-size : the size of the send buffer in kilobytes ( default 8 kilobytes) """ import datetime @@ -68,7 +69,7 @@ class Global(): opt = {'-d': [], '-D': False, '-v': False, '-V': False, '-r': '', '-B': 1024 * 32, '-S': 0, '-h': [], '-f': None, '--duration': 15, '--net': None, '--netserver': 'gpnetbenchServer', - '--netclient': 'gpnetbenchClient'} + '--netclient': 'gpnetbenchClient', '--buffer-size': 0} GV = Global() @@ -82,9 +83,9 @@ def strcmd(cmd): return reduce(lambda x, y: x + ' ' + y, map(lambda x: x.find(' ') > 0 and "'" + x + "'" or x, cmd)) -def gpssh(cmd): +def gpssh(cmd, verbose): c = ['%s/bin/gpssh' % GPHOME] - if GV.opt['-V']: + if verbose: c.append('-v') if GV.opt['-f']: c.append('-f') @@ -103,8 +104,8 @@ def gpssh(cmd): return not rc, out -def gpscp(src, dst): - c = ['%s/bin/gpscp' % GPHOME] +def gpsync(src, dst): + c = ['%s/bin/gpsync' % GPHOME] if GV.opt['-V']: c.append('-v') if GV.opt['-f']: @@ -133,26 +134,27 @@ def getPlatform(): def getMemory(): - if getPlatform() == 'linux': - ok, out = run("sh -c 'cat /proc/meminfo | grep MemTotal'") - if not ok: - return 0 + platform = getPlatform() + + if platform == 'linux': + rc, out = run("sh -c 'cat /proc/meminfo | grep MemTotal'") + if rc != 0: + return None word_list = out.strip().split(' ') val = int(word_list[len(word_list) - 2]) factor = word_list[len(word_list) - 1] if factor == 'kB': - return val * 1024 - return 0 + return val * 1024 if val else None - if getPlatform() == 'darwin': - ok, out = run("/usr/sbin/sysctl hw.physmem") - if not ok: - return 0 + if platform == 'darwin': + rc, out = run("/usr/sbin/sysctl hw.physmem") + if rc != 0: + return None word_list = out.strip().split(' ') val = int(word_list[1]) - return val + return val if val else None - return 0 + return None def parseMemorySize(line): @@ -202,7 +204,7 @@ def print_version(): def parseCommandLine(): try: - (options, args) = getopt.getopt(sys.argv[1:], '?vVDd:r:B:S:p:h:f:', ['duration=', 'version', 'netperf']) + (options, args) = getopt.getopt(sys.argv[1:], '?vVDd:r:B:S:p:h:f:', ['duration=', 'version', 'netperf', 'buffer-size=']) except Exception as e: usage('Error: ' + str(e)) exit(1) @@ -225,6 +227,8 @@ def parseCommandLine(): elif switch == '--netperf': GV.opt['--netserver'] = 'netserver' GV.opt['--netclient'] = 'netperf' + elif switch == '--buffer-size': + GV.opt[switch] = int(val) # run default tests (if not specified) if GV.opt['-r'] == '': @@ -261,7 +265,12 @@ def parseCommandLine(): usage('Error: maximum size for -B parameter is 1MB') if GV.opt['-S'] == 0: - GV.opt['-S'] = 2 * getMemory() / len(GV.opt['-d']) + system_mem_size = getMemory() + if system_mem_size is not None: + GV.opt['-S'] = 2 * system_mem_size / len(GV.opt['-d']) + else: + sys.exit('[Error] could not get system memory size. Instead, you can use the -S option to provide the file size value') + else: GV.opt['-S'] /= len(GV.opt['-d']) @@ -269,6 +278,14 @@ def parseCommandLine(): GV.opt['--duration'] = 15 print('[INFO] Invalid network duration specified. Using default (15 seconds)') + if GV.opt['--netclient'].find('netperf') >= 0: + if GV.opt['--buffer-size']: + print('[Warning] --buffer-size option will be ignored when the --netperf option is enabled') + else: + if GV.opt['--buffer-size'] <= 0: + print('[INFO] --buffer-size value is not specified or invalid. Using default (8 kilobytes)') + GV.opt['--buffer-size'] = 8 + # strip the last '/' from the dir dd = [] for d in GV.opt['-d']: @@ -320,13 +337,13 @@ def runTeardown(): for d in GV.opt['-d']: dirs = '%s %s/gpcheckperf_$USER' % (dirs, d) try: - gpssh('rm -rf ' + dirs) + gpssh('rm -rf ' + dirs, GV.opt['-V']) except: pass try: if GV.opt['--net']: - gpssh(killall(GV.opt['--netserver'])) + gpssh(killall(GV.opt['--netserver']), GV.opt['-V']) except: pass @@ -340,14 +357,14 @@ def runSetup(): print('--------------------') okCount = 0 try: - # check python reachable + # Verify python3 is accessible if GV.opt['-v']: - print('[Info] verify python interpreter exists') - (ok, out) = gpssh('python -c print') + print('[Info] verify python3 interpreter exists') + (ok, out) = gpssh('python3 -c print', GV.opt['-V']) if not ok: if not GV.opt['-v']: print(out) - sys.exit("[Error] unable to find python interpreter on some hosts\n" + sys.exit("[Error] unable to find python3 interpreter on some hosts\n" + " verify PATH variables on the hosts") # mkdir cperf @@ -358,7 +375,7 @@ def runSetup(): dirs = '%s %s/gpcheckperf_$USER' % (dirs, d) cmd = 'rm -rf %s ; mkdir -p %s' % (dirs, dirs) - (ok, out) = gpssh(cmd) + (ok, out) = gpssh(cmd, GV.opt['-V']) if not ok: print('failed gpssh: %s' % out) sys.exit("[Error] unable to make gpcheckperf directory. \n" @@ -391,12 +408,12 @@ def copyExecOver(fname): if not os.access(path, os.X_OK): sys.exit('[Exit] file not executable: ' + path) - (ok, out) = gpscp(path, '=:%s' % target) + (ok, out) = gpsync(path, '=:%s' % target) if not ok: - sys.exit('[Error] command failed: gpscp %s =:%s with output: %s' % (path, target, out)) + sys.exit('[Error] command failed: gpsync %s =:%s with output: %s' % (path, target, out)) # chmod +x file - (ok, out) = gpssh('chmod a+rx %s' % target) + (ok, out) = gpssh('chmod a+rx %s' % target, GV.opt['-V']) if not ok: sys.exit('[Error] command failed: chmod a+rx %s with output: %s' % (target, out)) @@ -414,7 +431,7 @@ def parseMultiDDResult(out): o = line[i + 2:] - if o.find('multidd total bytes ') > 0: + if o.find('multidd total bytes ') >= 0: h = line[1:i] o = o.split() m = re.search("(^\d+)", o[-1]) @@ -423,9 +440,10 @@ def parseMultiDDResult(out): bytes = int(m.group(1)) continue - if o.find('real') > 0: + if o.find('real') >= 0: h = line[1:i] o = o.split() + o[1] = o[1].replace(',', '.') m = re.search("(^\d+.\d+)", o[1]) if m is None: sys.exit('[Error] expected %s to be a floating point number' % o[1]) @@ -452,7 +470,7 @@ def runDiskWriteTest(multidd): cmd = cmd + (' -B %d' % GV.opt['-B']) if GV.opt['-S']: cmd = cmd + (' -S %d' % GV.opt['-S']) - (ok, out) = gpssh(cmd) + (ok, out) = gpssh(cmd, GV.opt['-V']) if not ok: sys.exit('[Error] command failed: %s with output: %s' % (cmd, out)) return parseMultiDDResult(out) @@ -471,7 +489,7 @@ def runDiskReadTest(multidd): cmd = cmd + (' -B %d' % GV.opt['-B']) if GV.opt['-S']: cmd = cmd + (' -S %d' % GV.opt['-S']) - (ok, out) = gpssh(cmd) + (ok, out) = gpssh(cmd, GV.opt['-V']) if not ok: sys.exit('[Error] command failed: %s with output: %s' % (cmd, out)) return parseMultiDDResult(out) @@ -484,7 +502,7 @@ def runStreamTest(): print('--------------------') cmd = copyExecOver('stream') - (ok, out) = gpssh(cmd) + (ok, out) = gpssh(cmd, GV.opt['-V']) if not ok: sys.exit('[Error] command failed: %s with output: %s' % (cmd, out)) out = io.StringIO(out) @@ -506,9 +524,9 @@ def startNetServer(): for i in range(5): if i > 0: print('[Warning] retrying with port %d' % port) - (ok, out) = gpssh(killall(GV.opt['--netserver'])) + (ok, out) = gpssh(killall(GV.opt['--netserver']), GV.opt['-V']) - (ok, out) = gpssh('%s -p %d > /dev/null 2>&1' % (rmtPath, port)) + (ok, out) = gpssh('%s -p %d > /dev/null 2>&1' % (rmtPath, port), GV.opt['-V']) if ok: return port @@ -534,14 +552,15 @@ def spawnNetperfTestBetween(x, y, netperf_path, netserver_port, sec=5): cmd = ('%s -H %s -p %d -t TCP_STREAM -l %s -f M -P 0 ' % (netperf_path, y, netserver_port, sec)) else: - cmd = ('%s -H %s -p %d -l %s -P 0 ' - % (netperf_path, y, netserver_port, sec)) + cmd = ('%s -H %s -p %d -l %s -P 0 -b %s' + % (netperf_path, y, netserver_port, sec, GV.opt['--buffer-size'])) + c = ['ssh', '-o', 'BatchMode yes', '-o', 'StrictHostKeyChecking no', x, cmd] proc = None try: - if GV.opt['-v']: + if GV.opt['-v'] or GV.opt['-V']: print('[Info]', strcmd(c)) proc = subprocess.Popen(c, stdout=subprocess.PIPE) except KeyboardInterrupt: @@ -739,14 +758,28 @@ def get_host_map(hostlist): seglist = dict() # segment list uniqhosts = dict() # unique host list - # get list of hostnames - rc, out = gpssh('hostname') + ''' + Get hostnames using non-verbose mode since verbose output makes parsing difficult with extra lines as show: + Using delaybeforesend 0.05 and prompt_validation_timeout 1.0 + [Reset ...] + [INFO] login sdw2 + [sdw2] sdw2 + [INFO] completed successfully + [Cleanup...] + ''' + rc, out = gpssh('hostname', False) + if not rc: raise Exception('Encountered error running hostname') + ''' Sample output: + [sdw1] sdw1 + [sdw2] sdw2 + ''' + # get unique hostname list for line in out.splitlines(): - seg, host = line.translate(None, '[]').split() + seg, host = line.translate(str.maketrans('','','[]')).split() uniqhosts[host] = seg # get list of segments associated with each host (can't use gpssh since it de-dupes hosts) @@ -755,7 +788,8 @@ def get_host_map(hostlist): proc = None try: - if GV.opt['-v']: print('[Info]', strcmd(cmd)) + if GV.opt['-v']: + print('[Info]', strcmd(cmd)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out = proc.stdout.read(-1) rc = proc.wait() @@ -781,7 +815,7 @@ def runNetPerfTestMatrix(): ''' (netperf, hostlist, netserver_port) = setupNetPerfTest() if not netperf: - return None + return None, None # dict() of seglist[segname] = hostname, uniqhosts[hostname] = 1 segment name seglist, uniqhosts = get_host_map(hostlist) @@ -813,13 +847,13 @@ def printMatrixResult(result, seglist): netTx = dict() netRx = dict() for h in result: - if netTx.has_key(h[0]): + if h[0] in netTx: netTx[h[0]] += float(h[6]) else: netTx[h[0]] = float(h[6]) # netRx requires that we lookup the hostname for a given segment name - if netRx.has_key(seglist[h[1]]): + if seglist[h[1]] in netRx: netRx[seglist[h[1]]] += float(h[6]) else: netRx[seglist[h[1]]] = float(h[6]) @@ -850,7 +884,7 @@ def printMatrixResult(result, seglist): copy = n[:] copy.sort() - median = copy[len(copy) / 2] + median = copy[len(copy) // 2] print('') print('Summary:') @@ -930,50 +964,55 @@ def printResult(title, result): print('') -try: - parseCommandLine() - runSetup() - diskWriteResult = diskReadResult = streamResult = netResult = None - tornDown = False +def main(): try: - if GV.opt['-r'].find('d') >= 0: - multidd = copyExecOver('multidd') - diskWriteResult = runDiskWriteTest(multidd) - diskReadResult = runDiskReadTest(multidd) + parseCommandLine() + runSetup() + diskWriteResult = diskReadResult = streamResult = netResult = None + tornDown = False + try: + if GV.opt['-r'].find('d') >= 0: + print('[Warning] Using %d bytes for disk performance test. This might take some time' % (GV.opt['-S'] * len(GV.opt['-d']))) + multidd = copyExecOver('multidd') + diskWriteResult = runDiskWriteTest(multidd) + diskReadResult = runDiskReadTest(multidd) - if GV.opt['-r'].find('s') >= 0: - streamResult = runStreamTest() + if GV.opt['-r'].find('s') >= 0: + streamResult = runStreamTest() - if GV.opt['--net'] == 'netperf': - netResult = runNetPerfTest() - elif GV.opt['--net'] == 'parallel': - netResult = runNetPerfTestParallel() - elif GV.opt['--net'] == 'matrix': - netResult, seglist = runNetPerfTestMatrix() + if GV.opt['--net'] == 'netperf': + netResult = runNetPerfTest() + elif GV.opt['--net'] == 'parallel': + netResult = runNetPerfTestParallel() + elif GV.opt['--net'] == 'matrix': + netResult, seglist = runNetPerfTestMatrix() - runTeardown() + runTeardown() + + finally: + print('') + print('====================') + print('== RESULT %s' % datetime.datetime.now().isoformat()) + print('====================') - finally: - print('') - print('====================') - print('== RESULT %s' % datetime.datetime.now().isoformat()) - print('====================') + if diskWriteResult: + printResult('disk write', diskWriteResult) - if diskWriteResult: - printResult('disk write', diskWriteResult) + if diskReadResult: + printResult('disk read', diskReadResult) - if diskReadResult: - printResult('disk read', diskReadResult) + if streamResult: + printResult('stream', streamResult) - if streamResult: - printResult('stream', streamResult) + if netResult and GV.opt['--net'] == 'matrix': + printMatrixResult(netResult, seglist) + elif netResult and GV.opt['--net']: + printNetResult(netResult) - if netResult and GV.opt['--net'] == 'matrix': - printMatrixResult(netResult, seglist) - elif netResult and GV.opt['--net']: - printNetResult(netResult) + runTeardown() - runTeardown() + except KeyboardInterrupt: + print('[Abort] Keyboard Interrupt ...') -except KeyboardInterrupt: - print('[Abort] Keyboard Interrupt ...') +if __name__ == '__main__': + main() diff --git a/gpMgmt/bin/gpexpand b/gpMgmt/bin/gpexpand index f9d0fe36b2b..3825095829a 100755 --- a/gpMgmt/bin/gpexpand +++ b/gpMgmt/bin/gpexpand @@ -442,7 +442,7 @@ class GpExpandStatus(): def _sync_status_file(self): """Syncs the gpexpand status file with the coordinator mirror""" - cpCmd = Scp('gpexpand copying status file to coordinator mirror', + cpCmd = Rsync('gpexpand copying status file to coordinator mirror', srcFile=self._status_filename, dstFile=self._status_standby_filename, dstHost=self._coordinator_mirror.getSegmentHostName()) @@ -512,7 +512,7 @@ class GpExpandStatus(): """ Sync the segment configuration backup file to standby """ if self._coordinator_mirror: self.logger.debug("Sync segment configuration backup file") - cpCmd = Scp('gpexpand copying segment configuration backup file to coordinator mirror', + cpCmd = Rsync('gpexpand copying segment configuration backup file to coordinator mirror', srcFile=self._gp_segment_configuration_backup, dstFile=self._segment_configuration_standby_filename, dstHost=self._coordinator_mirror.getSegmentHostName()) @@ -757,7 +757,7 @@ class SegmentTemplate: """Distributes template tar file to hosts""" for host in self.hosts: logger.debug('Copying tar file to %s' % host) - cpCmd = Scp(name='gpexpand distribute tar file to new hosts', + cpCmd = Rsync(name='gpexpand distribute tar file to new hosts', srcFile=self.schema_tar_file, dstFile=self.segTarDir, dstHost=host) @@ -846,7 +846,7 @@ class SegmentTemplate: localHostname = self.gparray.coordinator.getSegmentHostName() cmdName = 'gpexpand copying postgresql.conf to %s:%s/postgresql.conf' \ % (self.srcSegHostname, self.srcSegDataDir) - cpCmd = Scp(name=cmdName, srcFile=self.srcSegDataDir + '/postgresql.conf', + cpCmd = Rsync(name=cmdName, srcFile=self.srcSegDataDir + '/postgresql.conf', dstFile=self.tempDir, dstHost=localHostname, ctxt=REMOTE, remoteHost=self.srcSegHostname) cpCmd.run(validateAfter=True) @@ -854,7 +854,7 @@ class SegmentTemplate: self.logger.info('Copying pg_hba.conf from existing segment into template') cmdName = 'gpexpand copy pg_hba.conf to %s:%s/pg_hba.conf' \ % (self.srcSegHostname, self.srcSegDataDir) - cpCmd = Scp(name=cmdName, srcFile=self.srcSegDataDir + '/pg_hba.conf', + cpCmd = Rsync(name=cmdName, srcFile=self.srcSegDataDir + '/pg_hba.conf', dstFile=self.tempDir, dstHost=localHostname,ctxt=REMOTE, remoteHost=self.srcSegHostname) cpCmd.run(validateAfter=True) diff --git a/gpMgmt/bin/gpmemwatcher b/gpMgmt/bin/gpmemwatcher index 29895cbe75c..6569015bc09 100755 --- a/gpMgmt/bin/gpmemwatcher +++ b/gpMgmt/bin/gpmemwatcher @@ -171,7 +171,7 @@ def stopProcesses(host, workdir): return try: - subprocess.check_call('scp -q %s:%s/%s ./%s.%s' % (host, dest_dir, ps_file, host, ps_file), shell=True) + subprocess.check_call('rsync -q %s:%s/%s ./%s.%s' % (host, dest_dir, ps_file, host, ps_file), shell=True) except subprocess.CalledProcessError as e: print('Error retrieving data from host: ' + host, file=sys.stderr) print(e) diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index 46d2f636b7d..8aa208ba576 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1169,8 +1169,8 @@ def distribute_tarball(queue,list,tarball): hostname = db.getSegmentHostName() datadir = db.getSegmentDataDirectory() (head,tail)=os.path.split(datadir) - scp_cmd=Scp(name="copy coordinator",srcFile=tarball,dstHost=hostname,dstFile=head) - queue.addCommand(scp_cmd) + rsync_cmd=Rsync(name="copy coordinator",srcFile=tarball,dstHost=hostname,dstFile=head) + queue.addCommand(rsync_cmd) queue.join() queue.check_results() logger.debug("distributeTarBall finished") diff --git a/gpMgmt/bin/gppylib/commands/unix.py b/gpMgmt/bin/gppylib/commands/unix.py index 361bac0f69b..3d0b0582d14 100644 --- a/gpMgmt/bin/gppylib/commands/unix.py +++ b/gpMgmt/bin/gppylib/commands/unix.py @@ -501,7 +501,7 @@ def filedir_exists(self): return (not self.results.rc) -# -------------scp------------------ +# -------------rsync------------------ # MPP-13617 def canonicalize(addr): @@ -510,10 +510,10 @@ def canonicalize(addr): return '[' + addr + ']' -class Scp(Command): +class Rsync(Command): def __init__(self, name, srcFile, dstFile, srcHost=None, dstHost=None, recursive=False, ctxt=LOCAL, remoteHost=None): - cmdStr = findCmdInPath('scp') + " " + cmdStr = findCmdInPath('rsync') + " " if recursive: cmdStr = cmdStr + "-r " diff --git a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py index 91c537e4ffa..4bfe87ecd13 100644 --- a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py @@ -18,7 +18,7 @@ from gppylib.operations.utils import ParallelOperation, RemoteOperation from gppylib.system import configurationInterface as configInterface from gppylib.commands.gp import is_pid_postmaster, get_pid_from_remotehost -from gppylib.commands.unix import check_pid_on_remotehost, Scp +from gppylib.commands.unix import check_pid_on_remotehost from gppylib.programs.clsRecoverSegment_triples import RecoveryTriplet diff --git a/gpMgmt/bin/gppylib/operations/package.py b/gpMgmt/bin/gppylib/operations/package.py index 7ba8d34affa..f1e73563846 100644 --- a/gpMgmt/bin/gppylib/operations/package.py +++ b/gpMgmt/bin/gppylib/operations/package.py @@ -11,7 +11,7 @@ from gppylib import gplog from gppylib.commands import gp from gppylib.commands.base import Command, REMOTE, WorkerPool, ExecutionError - from gppylib.commands.unix import Scp + from gppylib.commands.unix import Rsync from gppylib.gpversion import GpVersion from gppylib.mainUtils import ExceptionNoStackTraceNeeded from gppylib.operations import Operation @@ -302,7 +302,7 @@ class RemoteCommand(Operation): """ DEPRECATED - TODO: AK: Rename as GpSsh, like GpScp below. + TODO: AK: Rename as GpSsh, like GpRsync below. """ def __init__(self, cmd_str, host_list): @@ -1019,7 +1019,7 @@ def execute(self): for package in install_package_set: logger.debug('copying %s to %s' % (package, self.host)) dstFile = os.path.join(GPHOME, package) - Scp(name='copying %s to %s' % (package, self.host), + Rsync(name='copying %s to %s' % (package, self.host), srcFile=os.path.join(GPPKG_ARCHIVE_PATH, package), dstFile=dstFile, dstHost=self.host).run(validateAfter=True) @@ -1072,12 +1072,12 @@ def execute(self): if linux_distribution_id() == 'ubuntu': # install package on segments if self.segment_host_list: - GpScp(srcFile, dstFile, self.segment_host_list).run() + GpRsync(srcFile, dstFile, self.segment_host_list).run() HostOperation(InstallDebPackageLocally(dstFile), self.segment_host_list).run() # install package on standby if self.standby_host: - Scp(name='copying %s to %s' % (srcFile, self.standby_host), + Rsync(name='copying %s to %s' % (srcFile, self.standby_host), srcFile=srcFile, dstFile=dstFile, dstHost=self.standby_host).run(validateAfter=True) @@ -1088,12 +1088,12 @@ def execute(self): else: # install package on segments if self.segment_host_list: - GpScp(srcFile, dstFile, self.segment_host_list).run() + GpRsync(srcFile, dstFile, self.segment_host_list).run() HostOperation(InstallPackageLocally(dstFile), self.segment_host_list).run() # install package on standby if self.standby_host: - Scp(name='copying %s to %s' % (srcFile, self.standby_host), + Rsync(name='copying %s to %s' % (srcFile, self.standby_host), srcFile=srcFile, dstFile=dstFile, dstHost=self.standby_host).run(validateAfter=True) @@ -1408,14 +1408,14 @@ def execute(self): # distribute package to segments srcFile = self.gppkg.abspath dstFile = os.path.join(GPHOME, self.gppkg.pkg) - GpScp(srcFile, dstFile, self.segment_host_list).run() + GpRsync(srcFile, dstFile, self.segment_host_list).run() # update package on segments HostOperation(UpdatePackageLocally(dstFile), self.segment_host_list).run() # update package on standby if self.standby_host: - Scp(name='copying %s to %s' % (srcFile, self.standby_host), + Rsync(name='copying %s to %s' % (srcFile, self.standby_host), srcFile=srcFile, dstFile=dstFile, dstHost=self.standby_host).run(validateAfter=True) @@ -1552,7 +1552,7 @@ def execute(self): logger.info('The package migration has completed.') -class GpScp(Operation): +class GpRsync(Operation): """ TODO: AK: This obviously does not belong here. My preference would be that it remain here until the following problem is solved. @@ -1562,14 +1562,14 @@ class GpScp(Operation): I suggest: We consume an extra parameter 'fanout'. We partition the host_list into a number of buckets - given by 'fanout'. For each bucket, we scp the artifact to the first host in the bucket, and then - we recursively invoke GpScp on that machine for the remaining hosts in its bucket. + given by 'fanout'. For each bucket, we rsync the artifact to the first host in the bucket, and then + we recursively invoke GpRsync on that machine for the remaining hosts in its bucket. - GpScp := ParallelOperation([ A(i) for i in range(0, n) ]) + GpRsync := ParallelOperation([ A(i) for i in range(0, n) ]) A := SerialOperation(B, C) - B := scp source_path target_path @ host_i + B := rsync source_path target_path @ host_i where host_i := the first host in the ith bucket - C := RemoteOperation(GpScp(target_path, target_path, host_list_i)) + C := RemoteOperation(GpRsync(target_path, target_path, host_list_i)) where host_list_i := the remaining hosts in the ith bucket """ @@ -1582,7 +1582,7 @@ def __init__(self, source_path, target_path, host_list): def execute(self): self.pool = WorkerPool() for host in self.host_list: - self.pool.addCommand(Scp(name='copying %s to %s' % (self.source_path, host), + self.pool.addCommand(Rsync(name='copying %s to %s' % (self.source_path, host), srcFile=self.source_path, dstFile=self.target_path, dstHost=host)) diff --git a/gpMgmt/bin/gppylib/operations/test/regress/test_package/__init__.py b/gpMgmt/bin/gppylib/operations/test/regress/test_package/__init__.py index 692ca12d6e0..e5c8c0d8f80 100644 --- a/gpMgmt/bin/gppylib/operations/test/regress/test_package/__init__.py +++ b/gpMgmt/bin/gppylib/operations/test/regress/test_package/__init__.py @@ -10,11 +10,11 @@ from gppylib.gpversion import MAIN_VERSION from contextlib import closing from gppylib.commands import gp -from gppylib.commands.unix import Scp +from gppylib.commands.unix import Rsync from gppylib.commands.base import Command, ExecutionError, REMOTE from gppylib.operations import Operation from gppylib.operations.unix import CheckFile, CheckRemoteFile, RemoveRemoteFile -from gppylib.operations.package import dereference_symlink, GpScp, linux_distribution_id, linux_distribution_version +from gppylib.operations.package import dereference_symlink, GpRsync, linux_distribution_id, linux_distribution_version from gppylib.commands.base import Command, REMOTE def get_os(): diff --git a/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals.py b/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals.py index 011e4fd996a..38ceb55cda7 100755 --- a/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals.py +++ b/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals.py @@ -5,9 +5,9 @@ import shutil from contextlib import closing -from gppylib.commands.unix import Scp +from gppylib.commands.unix import Rsync from gppylib.commands.base import ExecutionError -from gppylib.operations.package import GpScp +from gppylib.operations.package import GpRsync from gppylib.operations.unix import RemoveRemoteFile from gppylib.operations.test.regress.test_package import GppkgTestCase, unittest, get_host_list, ARCHIVE_PATH, RPM_DATABASE, run_command, skipIfSingleNode @@ -132,7 +132,7 @@ def test06_delete_package_from_archive_on_segment_and_install(self): try: self.install(gppkg_file) except ExecutionError as e: - Scp(name = "copy gppkg to segment", + Rsync(name = "copy gppkg to segment", srcFile = gppkg_file, dstFile = archive_file, srcHost = None, @@ -159,7 +159,7 @@ def test07_delete_package_from_archive_on_segment_and_uninstall(self): try: self.remove(gppkg_file) except ExecutionError as e: - GpScp(source_path = gppkg_file, + GpRsync(source_path = gppkg_file, target_path = archive_file, host_list = segment_host_list).run() self.fail("ExecutionError %s" % str(e)) @@ -187,7 +187,7 @@ def test08_uninstall_rpm_on_segments_and_install(self): #Install the rpm with closing(tarfile.open(self.alpha_spec.get_filename())) as tf: tf.extract(self.A_spec.get_filename()) - Scp(name = "copy rpm to segment", + Rsync(name = "copy rpm to segment", srcFile = self.A_spec.get_filename(), dstFile = self.A_spec.get_filename(), srcHost = None, @@ -229,7 +229,7 @@ def test10_install_rpm_on_segments_and_install(self): #Install the rpm with closing(tarfile.open(self.alpha_spec.get_filename())) as tf: tf.extract(self.A_spec.get_filename()) - Scp(name = "copy rpm to segment", + Rsync(name = "copy rpm to segment", srcFile = self.A_spec.get_filename(), dstFile = self.A_spec.get_filename(), srcHost = None, @@ -255,7 +255,7 @@ def test11_install_rpm_on_segments_and_uninstall(self): #Install the rpm with closing(tarfile.open(self.alpha_spec.get_filename())) as tf: tf.extract(self.A_spec.get_filename()) - Scp(name = "copy rpm to segment", + Rsync(name = "copy rpm to segment", srcFile = self.A_spec.get_filename(), dstFile = self.A_spec.get_filename(), srcHost = None, diff --git a/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals_on_standby.py b/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals_on_standby.py index 1e8bd93a73f..e7fcf823688 100755 --- a/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals_on_standby.py +++ b/gpMgmt/bin/gppylib/operations/test/regress/test_package/test_regress_muck_with_internals_on_standby.py @@ -5,8 +5,8 @@ from contextlib import closing from gppylib.commands.base import ExecutionError -from gppylib.commands.unix import Scp -from gppylib.operations.package import GpScp +from gppylib.commands.unix import Rsync +from gppylib.operations.package import GpRsync from gppylib.operations.test.regress.test_package import GppkgTestCase, unittest, skipIfNoStandby, get_host_list, ARCHIVE_PATH, run_command from gppylib.operations.unix import RemoveRemoteFile @@ -25,7 +25,7 @@ def test00_delete_package_from_archive_on_standby_and_install(self): try: self.install(gppkg_file) except ExecutionError as e: - Scp(name = "copy gppkg to standby", + Rsync(name = "copy gppkg to standby", srcFile = gppkg_file, dstFile = archive_file, srcHost = None, @@ -48,10 +48,10 @@ def test01_delete_package_from_archive_on_standby_and_uninstall(self): try: self.remove(gppkg_file) except ExecutionError as e: - GpScp(source_path = gppkg_file, + GpRsync(source_path = gppkg_file, target_path = archive_file, host_list = get_host_list()[1]).run() - Scp(name = "copy gppkg to standby", + Rsync(name = "copy gppkg to standby", srcFile = gppkg_file, dstFile = archive_file, srcHost = None, @@ -73,7 +73,7 @@ def test02_uninstall_rpm_on_standby_and_install(self): #Install the rpm with closing(tarfile.open(self.alpha_spec.get_filename())) as tf: tf.extract(self.A_spec.get_filename()) - Scp(name = "copy rpm to standby", + Rsync(name = "copy rpm to standby", srcFile = self.A_spec.get_filename(), dstFile = self.A_spec.get_filename(), srcHost = None, @@ -101,7 +101,7 @@ def test04_install_rpm_on_standby_and_install(self): #Install the rpm with closing(tarfile.open(self.alpha_spec.get_filename())) as tf: tf.extract(self.A_spec.get_filename()) - Scp(name = "copy the rpm to standby", + Rsync(name = "copy the rpm to standby", srcFile = self.A_spec.get_filename(), dstFile = self.A_spec.get_filename(), srcHost = None, @@ -119,7 +119,7 @@ def test05_install_rpm_on_standby_and_uninstall(self): #Install the rpm with closing(tarfile.open(self.alpha_spec.get_filename())) as tf: tf.extract(self.A_spec.get_filename()) - Scp(name = "copy rpm to standby", + Rsync(name = "copy rpm to standby", srcFile = self.A_spec.get_filename(), dstFile = self.A_spec.get_filename(), srcHost = None, diff --git a/gpMgmt/bin/gppylib/operations/test/test_package.py b/gpMgmt/bin/gppylib/operations/test/test_package.py index d4f65648d12..202a53db17c 100755 --- a/gpMgmt/bin/gppylib/operations/test/test_package.py +++ b/gpMgmt/bin/gppylib/operations/test/test_package.py @@ -12,7 +12,7 @@ from gppylib.gparray import GpArray from contextlib import closing from gppylib.commands import gp -from gppylib.commands.unix import Scp +from gppylib.commands.unix import Rsync from gppylib.commands.base import Command, ExecutionError from gppylib.operations import Operation from gppylib.operations.unix import CheckFile, RemoveRemoteFile @@ -632,7 +632,7 @@ def test04_delete_package_from_archive_on_segment(self): try: self.install(gppkg_file) except ExecutionError as e: - Scp(name="copy to segment", srcFile=gppkg_file, dstFile=archive_file, srcHost=None, + Rsync(name="copy to segment", srcFile=gppkg_file, dstFile=archive_file, srcHost=None, dstHost=segment_host_list[0]).run(validateAfter=True) self.fail("ExecutionError %s" % e) @@ -653,7 +653,7 @@ def test05_delete_package_from_archive_on_standby(self): try: self.install(gppkg_file) except ExecutionError as e: - Scp(name="copy to segment", srcFile=gppkg_file, dstFile=archive_file, srcHost=None, dstHost=standby).run( + Rsync(name="copy to segment", srcFile=gppkg_file, dstFile=archive_file, srcHost=None, dstHost=standby).run( validateAfter=True) self.fail("ExecutionError %s" % e) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckperf.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckperf.py new file mode 100644 index 00000000000..b1de2f7896b --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckperf.py @@ -0,0 +1,101 @@ +import imp +import os +import sys +from mock import patch +from gppylib.test.unit.gp_unittest import GpTestCase,run_tests + +class GpCheckPerf(GpTestCase): + def setUp(self): + gpcheckcat_file = os.path.abspath(os.path.dirname(__file__) + "/../../../gpcheckperf") + self.subject = imp.load_source('gpcheckperf', gpcheckcat_file) + + def tearDown(self): + super(GpCheckPerf, self).tearDown() + + @patch('gpcheckperf.getPlatform', return_value='darwin') + @patch('gpcheckperf.run') + def test_get_memory_on_darwin(self, mock_run, mock_get_platform): + mock_run.return_value = [1, 'hw.physmem: 1234'] + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, None) + + mock_run.return_value = [0, 'hw.physmem: 0'] + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, None) + + mock_run.return_value = [0, 'hw.physmem: 1234'] + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, 1234) + + @patch('gpcheckperf.getPlatform', return_value='linux') + @patch('gpcheckperf.run') + def test_get_memory_on_linux(self, mock_run, mock_get_platform): + mock_run.return_value = [1, 'MemTotal: 10 kB'] + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, None) + + mock_run.return_value = [0, 'MemTotal: 0 kB'] + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, None) + + mock_run.return_value = [0, 'MemTotal: 10 kB'] + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, 10240) + + @patch('gpcheckperf.getPlatform', return_value='abc') + def test_get_memory_on_invalid_platform(self, mock_get_platform): + actual_result = self.subject.getMemory() + self.assertEquals(actual_result, None) + + @patch('gpcheckperf.getMemory', return_value=None) + def test_parseCommandLine_when_get_memory_fails(self, mock_get_memory): + sys.argv = ["gpcheckperf", "-h", "locahost", "-r", "d", "-d", "/tmp"] + with self.assertRaises(SystemExit) as e: + self.subject.parseCommandLine() + + self.assertEqual(e.exception.code, '[Error] could not get system memory size. Instead, you can use the -S option to provide the file size value') + + @patch('gpcheckperf.getMemory', return_value=123) + def test_parseCommandLine_when_get_memory_succeeds(self, mock_get_memory): + sys.argv = ["gpcheckperf", "-h", "locahost", "-r", "d", "-d", "/tmp"] + self.subject.parseCommandLine() + self.assertEqual(self.subject.GV.opt['-S'], 246.0) + + def test_parseMultiDDResult_when_output_regular(self): + inputText = """[localhost] dd if=/dev/zero of=/tmp/gpcheckperf_gpadmin/ddfile count=131072 bs=32768 +[localhost] 131072+0 records in +[localhost] 131072+0 records out +[localhost] 4294967296 bytes transferred in 2.973025 secs (1444645536 bytes/sec) +[localhost] +[localhost] multidd total bytes 4294967296 +[localhost] real 3.65 +[localhost] user 0.18 +[localhost] sys 2.52 + """ + actual_result = self.subject.parseMultiDDResult(inputText) + (mbps, time, bytes) = actual_result["localhost"] + exp_mbps, exp_time, exp_bytes = (1122.1917808219177, 3.65, 4294967296) + self.assertEqual(mbps, exp_mbps) + self.assertEqual(time, exp_time) + self.assertEqual(bytes, exp_bytes) + + def test_parseMultiDDResult_when_output_comma(self): + inputText = """[localhost] dd if=/dev/zero of=/tmp/gpcheckperf_gpadmin/ddfile count=131072 bs=32768 +[localhost] 131072+0 records in +[localhost] 131072+0 records out +[localhost] 4294967296 bytes transferred in 2.973025 secs (1444645536 bytes/sec) +[localhost] +[localhost] multidd total bytes 4294967296 +[localhost] real 3,65 +[localhost] user 0,18 +[localhost] sys 2,52 + """ + actual_result = self.subject.parseMultiDDResult(inputText) + (mbps, time, bytes) = actual_result["localhost"] + exp_mbps, exp_time, exp_bytes = (1122.1917808219177, 3.65, 4294967296) + self.assertEqual(mbps, exp_mbps) + self.assertEqual(time, exp_time) + self.assertEqual(bytes, exp_bytes) + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_package.py b/gpMgmt/bin/gppylib/test/unit/test_unit_package.py index 8ea90face04..a4d3288e37b 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_package.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_package.py @@ -183,7 +183,7 @@ def setUp(self): patch('gppylib.operations.package.MakeDir'), patch('gppylib.operations.package.CheckRemoteDir'), patch('gppylib.operations.package.MakeRemoteDir'), - patch('gppylib.operations.package.Scp'), + patch('gppylib.operations.package.Rsync'), patch('gppylib.operations.package.RemoteOperation'), patch('gppylib.operations.package.RemoveRemoteFile'), patch('gppylib.operations.package.InstallPackageLocally'), @@ -199,7 +199,7 @@ def setUp(self): self.mock_listdir = self.get_mock_from_apply_patch('listdir') self.mock_command = self.get_mock_from_apply_patch('Command') self.mock_logger = self.get_mock_from_apply_patch('logger') - self.mock_scp = self.get_mock_from_apply_patch('Scp') + self.mock_rsync = self.get_mock_from_apply_patch('Rsync') self.mock_install_packages_locally = self.get_mock_from_apply_patch('InstallPackageLocally') @@ -228,7 +228,7 @@ def test__execute_install_on_segments_when_package_are_missing(self): hostname = 'localhost' subject = SyncPackages(hostname) subject.execute() - self.assertEqual(self.mock_scp.call_count, 2) + self.assertEqual(self.mock_rsync.call_count, 2) self.assertEqual(self.make_dir_mock.call_count, 1) self.assertEqual(self.make_remote_dir_mock.call_count, 1) diff --git a/gpMgmt/bin/gpshrink b/gpMgmt/bin/gpshrink index 386c64a7e24..fb06e16a055 100644 --- a/gpMgmt/bin/gpshrink +++ b/gpMgmt/bin/gpshrink @@ -436,7 +436,7 @@ class gpshrinkStatus(): def _sync_status_file(self): """Syncs the gpshrink status file with the coordinator mirror""" - cpCmd = Scp('gpshrink copying status file to coordinator mirror', + cpCmd = Rsync('gpshrink copying status file to coordinator mirror', srcFile=self._status_filename, dstFile=self._status_standby_filename, dstHost=self._coordinator_mirror.getSegmentHostName()) @@ -503,7 +503,7 @@ class gpshrinkStatus(): """ Sync the segment configuration backup file to standby """ if self._coordinator_mirror: self.logger.debug("Sync segment configuration backup file") - cpCmd = Scp('gpshrink copying segment configuration backup file to coordinator mirror', + cpCmd = Rsync('gpshrink copying segment configuration backup file to coordinator mirror', srcFile=self._gp_segment_configuration_backup, dstFile=self._segment_configuration_standby_filename, dstHost=self._coordinator_mirror.getSegmentHostName()) diff --git a/gpMgmt/bin/gpssh-exkeys b/gpMgmt/bin/gpssh-exkeys index 1a1b2ec91dd..cf9853360fd 100755 --- a/gpMgmt/bin/gpssh-exkeys +++ b/gpMgmt/bin/gpssh-exkeys @@ -751,7 +751,7 @@ try: for h in GV.newHosts: - cmd = ('scp -q -o "BatchMode yes" -o "NumberOfPasswordPrompts 0" ' + + cmd = ('rsync -q -e "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0" ' + '%s %s %s %s %s:.ssh/ 2>&1' % (GV.authorized_keys_fname, GV.known_hosts_fname, @@ -792,7 +792,7 @@ try: remoteIdentity = GV.id_rsa_fname remoteIdentityPub = GV.id_rsa_pub_fname - cmd = ('scp -q -o "BatchMode yes" -o "NumberOfPasswordPrompts 0" ' + + cmd = ('rsync -q -e "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0" ' + '%s %s %s %s %s:.ssh/ 2>&1' % (remoteAuthKeysFile, remoteKnownHostsFile, diff --git a/gpMgmt/bin/gpscp b/gpMgmt/bin/gpsync similarity index 86% rename from gpMgmt/bin/gpscp rename to gpMgmt/bin/gpsync index 21be83e90d6..e656ca97a88 100755 --- a/gpMgmt/bin/gpscp +++ b/gpMgmt/bin/gpsync @@ -1,15 +1,16 @@ #!/usr/bin/env python3 ''' -gpscp -- scp to multiple hosts at once +gpsync -- rsync to multiple hosts at once -Usage: gpscp [--version] [-?v] [-r] [-p port] [-u user] +Usage: gpsync [--version] [-?v] [-r] [-a] [-p port] [-u user] [-h host] [-f hostfile] [-J host_substitution_character] [[user@]host1:]file1 [...] [[user@]hosts2:]file2 --version : print version information -? : print this help screen -v : verbose mode -r : recursively copy entire directories + -a : archive mode; equals -rlptgoD (no -H,-A,-X) -h host : ssh host to connect to (multiple -h is okay) -f file : a file listing all hosts to connect to -J character : character to be substitute as hostname [default='='] @@ -36,6 +37,7 @@ class Global: opt['-f'] = None opt['-J'] = '=:' opt['-r'] = False + opt['-a'] = False filePath = [] @@ -61,7 +63,7 @@ def print_version(): ############# def parseCommandLine(): try: - (options, args) = getopt.getopt(sys.argv[1:], '?vrJ:p:u:h:f:', ['version']) + (options, args) = getopt.getopt(sys.argv[1:], '?vraJ:p:u:h:f:', ['version']) except Exception as e: usage('Error: ' + str(e)) @@ -78,6 +80,8 @@ def parseCommandLine(): GV.opt[switch] = val + ':' elif (switch == '-r'): GV.opt[switch] = True + elif (switch == '-a'): + GV.opt[switch] = True elif (switch == '--version'): print_version() @@ -109,13 +113,14 @@ try: if len(GV.opt['-h']) == 0: usage('Error: missing hosts in -h and/or -f arguments') - scp = 'scp -o "BatchMode yes" -o "StrictHostKeyChecking no"' - if GV.opt['-r']: scp += ' -r' + rsync = 'rsync -e "ssh -o BatchMode=yes -o StrictHostKeyChecking=no"' + if GV.opt['-r']: rsync += ' -r' + if GV.opt['-a']: rsync += ' -a' proc = [] for peer in GV.opt['-h']: peer = canonicalize_address(peer) # MPP-13617 - cmd = scp + ' ' + cmd = rsync + ' ' for f in GV.filePath: cmd += f.replace(GV.opt['-J'], '%s:' % peer) + ' ' if GV.opt['-v']: print('[INFO]', cmd) diff --git a/gpMgmt/bin/lib/gp_bash_functions.sh b/gpMgmt/bin/lib/gp_bash_functions.sh index dc996cb6411..e23f7d9f5ad 100755 --- a/gpMgmt/bin/lib/gp_bash_functions.sh +++ b/gpMgmt/bin/lib/gp_bash_functions.sh @@ -87,7 +87,7 @@ MV=`findCmdInPath mv` MKDIR=`findCmdInPath mkdir` PING=`findCmdInPath ping` RM=`findCmdInPath rm` -SCP=`findCmdInPath scp` +RSYNC=`findCmdInPath rsync` SED=`findCmdInPath sed` SLEEP=`findCmdInPath sleep` SORT=`findCmdInPath sort` @@ -158,7 +158,7 @@ PG_CONF=postgresql.conf PG_INTERNAL_CONF=internal.auto.conf PG_HBA=pg_hba.conf if [ x"$TRUSTED_SHELL" = x"" ]; then TRUSTED_SHELL="$SSH"; fi -if [ x"$TRUSTED_COPY" = x"" ]; then TRUSTED_COPY="$SCP"; fi +if [ x"$TRUSTED_COPY" = x"" ]; then TRUSTED_COPY="$RSYNC "; fi PG_CONF_ADD_FILE=$WORKDIR/postgresql_conf_gp_additions DEFAULTDB=template1 ETCD_CONFIG_TMP_FILE=/tmp/cbdb_etcd.conf diff --git a/gpMgmt/doc/gpcheckperf_help b/gpMgmt/doc/gpcheckperf_help index 23820d46ab1..8fd95568530 100755 --- a/gpMgmt/doc/gpcheckperf_help +++ b/gpMgmt/doc/gpcheckperf_help @@ -13,7 +13,7 @@ gpcheckperf -d [-d ...] gpcheckperf -d {-f | -h [-h ...]} - [ -r n|N|M [--duration