D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
proc
/
2
/
cwd
/
opt
/
dedrads
/
Filename :
first_setup.py
back
Copy
#!/usr/lib/rads/venv/bin/python3 ''' script to resolve known issues with new vps/dedi and improve thier default configurations it is to be ran with -c on provision to alert of issues via STR/stdout -c is fast, but without flag does all the stuff that is slow (~30s) will not send STRs unless the -s flag is used Author: MathewL ''' import shlex import subprocess as sp import os import sys import re import math import shutil import argparse import socket import rads class FirstSetup: def __init__(self): self.str_msg = '' self.run_log = '' self.get_args() self.is_a_vps() self.get_mem_total() self.host_name = socket.gethostname() def get_args(self): ''' argz ''' parser = argparse.ArgumentParser() parser.add_argument( '-s', '--send-str', action='store_true', default=False, help=('Silence most output, sends STR to T2S for errors.\n'), ) parser.add_argument( '-n', '--no-ea4', action='store_true', default=False, help=('Skips installing missing apache modules\n'), ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help=('Enables verbose output with -s\n'), ) parser.add_argument( '-c', '--check', action='store_true', default=False, help=('Performs the new server checks instead of tweaks\n'), ) parser.add_argument( '-a', '--all', action='store_true', default=False, help=('Does both server tweaks\n'), ) parser.add_argument( '-d', '--dns-cluster', action='store_true', default=False, help=('Checks dns clustering\n'), ) self.args = parser.parse_args() def run_cmd(self, cmd, shell=False): ''' runs subprocess, builds log var, returns stdout ''' if shell: input_str = cmd else: input_str = shlex.join(cmd) self.log(f'EXEC: {input_str}\n') try: ret = sp.run( cmd, stdout=sp.PIPE, text=True, shell=shell, check=False ) except Exception as e: error_str = f'Error on cmd:\n{input_str}\nOutput:\n{e}\n' self.log(error_str) self.log_str(error_str) result = ret.stdout self.log(f'OUT: {result}\n', end=True) return result def make_str(self): ''' create an STR ticket ''' if self.is_vps: server_string = 'VPS' else: server_string = 'Dedicated Server' try: rads.make_ticket( dest='str@imhadmin.net', subject=f"New {server_string} with errors: {self.host_name}", body=f"""Errors encountered during first_setup.py: Host: {self.host_name} {self.str_msg} """, ) except rads.TicketError as exc: self.log(f"Failed to create STR ticket - {exc}") def final_output(self): ''' finishes program and handles final output (str/stdout) ''' if self.str_msg and self.args.send_str: self.log('creating str\n') self.make_str() if not self.str_msg and self.args.send_str: sys.exit(0) print(self.str_msg or 'first_setup complete.') sys.exit(0) def log(self, line, end=False): ''' logging function ''' if not line: line = '\n' if not self.args.send_str or self.args.verbose: print(line, end='') self.run_log += line if end: bar = '-' * 10 + '\n' # pylint: disable=disallowed-name self.run_log += bar if not self.args.send_str or self.args.verbose: print(bar, end='') def log_str(self, msg): self.str_msg += '-' * 10 + '\n' self.str_msg += msg def refresh_services(self): ''' runs after user provisioning ''' self.run_cmd(('package-cleanup', '-y', '--cleandupes')) # make sure rsyslog is working os.remove('/var/lib/rsyslog/imjournal.state') self.run_cmd(('systemctl', 'restart', 'rsyslog')) # fix quotas, takes less than a second usually on new cts self.run_cmd(('/scripts/fixquotas',)) # ensures systemd-logind not causing ssh slowness self.run_cmd(('systemctl', 'restart', 'systemd-logind')) def check_clustering(self): output = self.run_cmd(('du', '-s', '/var/cpanel/cluster')) if output.startswith('4\t'): self.log_str( 'Error: DNS clustering is not set up, ' '/var/cpanel/cluster is empty\n' ) def check_things(self): ''' checks a few things so the customer wont have to contact us for them makes an str if issues are found and the -s flag is used (vps auto prov) otherwise the errors go to stdout (dedi manual prov) intended to run after user provisioning. ''' # checks for A record for hostname output = self.run_cmd(('dig', '+short', '@ns', self.host_name)) if not output: self.log_str('Error: A record for hostname not found\n') # checks if dns clustering is unconfigured # in 15 minutes delay = '15' if self.args.dns_cluster: flags = '-ds' else: flags = '-d' p_cmd = f'/opt/dedrads/first_setup.py {flags}' p_cmd = p_cmd.encode('utf-8') with sp.Popen( ('at', 'now', '+', delay, 'minutes'), stdin=sp.PIPE, stdout=sp.DEVNULL, stderr=sp.PIPE, # encoding='utf-8', ) as proc: _, stderr = proc.communicate(p_cmd) if proc.poll(): self.log_str( 'Failed to schedule DNS cluster check with atd. ' f'Error:{stderr}' ) tweak_cmd = ( 'whmapi1', 'set_tweaksetting', 'key=mycnf_auto_adjust_maxallowedpacket', 'value=0', ) ssl_cmd = ( '/usr/local/cpanel/bin/checkallsslcerts', '--allow-retry', '--verbose', ) # check cpanel license output = self.run_cmd(('/usr/local/cpanel/cpkeyclt',)) if 'Update succeeded' not in output: self.log_str( 'Error: New server with invalid ' 'cPanel license.\nstdout:\n{}' 'Run these commands after fixing cPanel the license:\n' '{}\n{}\n'.format( output, ' '.join(tweak_cmd), ' '.join(ssl_cmd), ) ) else: # these require a valid cPanel license to run self.run_cmd(ssl_cmd) self.run_cmd(tweak_cmd) # restarts cpanel to get new service ssl if they were bad and # are now signed self.run_cmd('service cpanel restart &>/dev/null', shell=True) def tune_system(self): ''' These operations are kinda slow (30 seonds or so) So they are best to be ran before the customer pays They also do not require a valid cpanel license ''' if os.path.exists('/usr/sbin/yum-complete-transaction'): self.run_cmd(('/usr/sbin/yum-complete-transaction', '-y')) if self.is_vps: # Perl dependancies for IMH scripts self.run_cmd( ( 'yum', '-y', 'install', 'perl-File-Slurp', 'perl-YAML', 'perl-YAML-Syck', 'perl-Template-Toolkit', ) ) self.run_cmd( ( 'yes y | cpan -i Switch ' 'LWP::Protocol::https ' 'IO::Scalar ' 'Date::Parse ' 'Text::Template ' 'CDB_File &>/dev/null' ), shell=True, ) else: # for dedis self.run_cmd(('cpan', '-i', 'JSON::XS')) self.run_cmd(('kcarectl', '-u')) def set_swappiness(self): '''Sets swappiness to 0 and clears swap if dedicated server''' if self.is_vps: return self.run_cmd(['sysctl', '-w', 'vm.swappiness=0']) self.log('Swap is now clearing\n') self.run_cmd(['swapoff', '-a']) self.run_cmd(['swapon', '-a']) def tune_ea4(self): ''' Installs missing Apache modules ''' if self.args.no_ea4: return ea_file = '/etc/cpanel/ea4/profiles/custom/imh.json' if not os.path.isfile(ea_file): self.log( ( 'missing ea4 profile, skipping ' 'apache module install. path:\n{}\n'.format(ea_file) ), end=True, ) ea4_tup = ( 'ea-apache24-mod_headers', 'ea-apache24-mod_env', 'ea-apache24-mod_suexec', 'ea-apache24-mod_version', 'ea-apache24-mod_proxy_fcgi', 'ea-apache24-mod_remoteip', ) after = '"ea-apache24",' for value in ea4_tup: search = fr'''\s*("|')?{value}("|')?,''' value = f''' "{value}",''' self.conf_set(search, value, ea_file, after=after) self.run_cmd( ('/usr/local/bin/ea_install_profile', '--install', ea_file) ) def tune_apache_php(self): '''Does math to figure out Apache settings''' min_spare = 5 max_spare = 20 server_limit = int(math.floor(self.mem_total / 46000 / 4)) if server_limit > 150: server_limit = 150 elif server_limit < 15: server_limit = 10 # https://httpd.apache.org/docs/2.4/mod/worker.html # ServerLimit is a hard limit on the number of active child processes # and must be greater than or equal to the MaxRequestWorkers directive # divided by the ThreadsPerChild directive." # server_limit >= max_req / 25 # max_req <= server_limit * 25 max_workers = server_limit * 25 # tested only on the new file from cpanel 82+, might work on the old # local file though conf_file = '/etc/cpanel/ea4/ea4.conf' self.conf_set( r'(^\s*"minspareservers"\s*:\s*"?)\d+("?)[^$]', fr'\g<1>{min_spare}\g<2>,', conf_file, fallback=False, ) self.conf_set( r'(^\s*"maxspareservers"\s*:\s*"?)\d+("?)[^$]', fr'\g<1>{max_spare}\g<2>,', conf_file, fallback=False, ) self.conf_set( r'(^\s*"serverlimit"\s*:\s*"?)\d+("?)[^$]', fr'\g<1>{server_limit}\g<2>,', conf_file, fallback=False, ) self.conf_set( r'(^\s*"maxclients"\s*:\s*"?)\d+("?)[^$]', fr'\g<1>{max_workers}\g<2>,', conf_file, fallback=False, ) if self.is_vps: self.log('Setting FPM default pool settings\n') # Removed max_children settings due to potential resource abuse. # Only raises idle timeout and max requests # Current default max req is 20, timeout is 10 # Leaving max children as default fpm_config = 'pm_process_idle_timeout : 30\npm_max_requests : 128\n' try: os.makedirs('/var/cpanel/ApachePHPFPM') except (OSError, FileExistsError) as e: self.log(f"{e}\n") self.log('Setting Apache global config\n') path = '/var/cpanel/ApachePHPFPM/system_pool_defaults.yaml' with open(path, 'w', encoding='utf-8') as fh: fh.write(fpm_config) else: self.log('Skipping FPM default pool settings on Dedicated\n') self.run_cmd(['/scripts/rebuildhttpdconf']) self.run_cmd(['/scripts/php_fpm_config', '--rebuild']) self.run_cmd(['/scripts/restartsrv_apache_php_fpm']) self.run_cmd('/scripts/restartsrv_apache &>/dev/null', shell=True) def tune_mysql(self): ''' sets innodb buffer pool size and instances maxes out at 4GB of innodb buffer pool every 1GB gets another pool instance ''' mem_gate = 0 pool_count = 1 pool_size = 2 mem_tiers = ( (1000000, 1, 256), (3000000, 1, 512), (7000000, 1, 1024), (14000000, 2, 2048), (30000000, 4, 4096), ) buffer_pool = 256 pool_instances = 1 for tier in mem_tiers: if self.mem_total > tier[mem_gate]: buffer_pool = tier[pool_size] pool_instances = tier[pool_count] else: break buffer_pool = f'innodb_buffer_pool_size={buffer_pool}M\n' pool_instances = f'innodb_buffer_pool_instances={pool_instances}\n' conf_file = '/etc/my.cnf' self.conf_set( r'^\s*innodb_buffer_pool_size\s*=[^$]*', buffer_pool, conf_file ) self.conf_set( r'^\s*innodb_buffer_pool_instances\s*=[^$]*', pool_instances, conf_file, ) self.run_cmd(['service', 'mysql', 'restart']) def get_mem_total(self): ''' Gets total memory switched to free because /proc/meminfo may not be kb in the future ''' cmd = '''free |sed -e 1d -e 3d | awk '{print $2}' ''' mem_total = self.run_cmd(cmd, shell=True) mem_total = int(mem_total) if not mem_total: self.log('Failed to check available memory\n', end=True) sys.exit(1) self.mem_total = mem_total def conf_set( self, _search, _replace, _file, after='', multiple=False, fallback=True ): ''' configuration management function ''' after_found = 0 replace_count = 0 base_fn = os.path.basename(_file) tmp_fn = f'/tmp/{base_fn}.tmp' with open(tmp_fn, 'w', encoding='utf-8') as tmp_fh: with open(_file, encoding='utf-8') as og_fh: og_fh = og_fh.read() # checks if already set, making it re-runnable if _replace in og_fh: self.log( f'Setting already set in file {_file}:\n{_replace}\n', end=True, ) return og_fh = og_fh.splitlines(True) for og_line in og_fh: # detects the after line if after and not after_found and re.search(after, og_line): after_found = 1 self.log(f'after line found: {after}\n') # main conditional gate for doing the replace if ( ( (not after and _search != 'APPEND') or (after and after_found) ) and (multiple or not replace_count) and (re.search(_search, og_line) or _search == 'APPEND') ): if _search == 'APPEND': self.log( f'appending {_replace} after {after}\n', end=True, ) new_line = f'{og_line}{_replace}\n' tmp_fh.write(new_line) replace_count += 1 continue self.log( 'doing traditional search/replace:\n' f'{_search} -> {_replace}\n', end=True, ) new_line = re.sub(_search, _replace, og_line) tmp_fh.write(new_line) replace_count += 1 continue tmp_fh.write(og_line) if not replace_count and fallback: if not after: self.log( 'appending to end of file as fallback:\n{}\n'.format( _replace ), end=True, ) replace_count += 1 tmp_fh.write(f'{_replace}\n') else: self.log('doing recurisve conf_set for after fallback\n') os.remove(tmp_fn) self.conf_set( 'APPEND', _replace, _file, after=after, fallback=False ) return if not replace_count: self.log( ( 'Warning: Failed to replace (fallback False):\n' 'search:{}\nreplace:{}\n' 'file:{}\nafter:{}\n'.format( _search, _replace, _file, after ) ), end=True, ) os.remove(tmp_fn) return shutil.move(tmp_fn, _file) def is_a_vps(self): if os.path.exists('/proc/vz/veinfo'): self.is_vps = True else: self.is_vps = False def main(): """if -c is used, the first section runs this is intended to be ran after the customers user is added without -c is meant to be ran before the customer is provisioned (clonevps /ready dedi)""" setup = FirstSetup() if setup.args.all: # does all the config changes setup.set_swappiness() setup.tune_system() setup.tune_ea4() setup.refresh_services() setup.check_things() setup.tune_apache_php() setup.tune_mysql() elif setup.args.check: # requires cpanel license / vps package must be set / is fast setup.refresh_services() setup.check_things() setup.tune_apache_php() setup.tune_mysql() elif setup.args.dns_cluster: setup.check_clustering() else: # stuff to be performed on clones/ready dedis before purchase setup.set_swappiness() setup.tune_system() setup.tune_ea4() setup.final_output() if __name__ == '__main__': main()