from fabric.api import run
from fabric.api import hosts, local, settings, abort
from fabric.state import env

import os

crawler_nodes = ('yegiic', 'yegiic4', 'yegiic5', 'yegiic6', 'yegiic7', 'yegiic8')

def proxy_restart():
""" Pull latest configuration and restart proxy nodes """

for host in ('proxylb','proxylb2', 'proxylb3'):
env.host_string = host
env.user = 'ubuntu'
run('cd ~/proxyrotate; git pull -v; source ~/proxy/bin/activate; python rotate_proxies.py -w; python rotate_proxies.py -W; sudo service haproxy restart')

def proxy_monitor_restart():
""" Pull latest configuration and restart proxy nodes """

for host in ('proxylb','proxylb2', 'proxylb3'):
env.host_string = host
env.user = 'ubuntu'
with settings(warn_only=True):
run('cd ~/proxyrotate; source ~/proxy/bin/activate; pkill -KILL -f proxy_monitor_restart; python proxy_monitor_restart.py')

def git_pull():
for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Syncing code on',host,'...'
run('cd ~/yegii-crawler/crawlers; git pull -v')

def kill_all_crawls():
for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Killing all crawls...'
run('pkill -KILL -f run_spider_proc')

def kill_crawl():

with settings(warn_only=True):
env.user = 'ubuntu'
crawl_id = raw_input('Enter crawl id: ').strip()
print 'Killing crawl id',crawl_id
run('cd ~/yegii-crawler/crawlers/; ~/yegi/bin/python mark_done.py %s 1' % crawl_id)

def free_buffers():

env.user = 'ubuntu'
run('sudo su -c "echo 3 > /proc/sys/vm/drop_caches"; free -m')

def remove_crawl_logs():

for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Dropping crawl logs older than 3 days on',host
run('cd ~/yegii-crawler/crawlers/logs/; find *.log -mtime +3 | xargs rm -f -;')

def apply_firewall():

for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Syncing code on',host,'...'
run('cd ~/yegii-crawler/; cp iptables.rules ~; sudo iptables-restore < iptables.rules')

def restart_crawlers():
for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Syncing code on',host,'...'
run('cd ~/yegii-crawler/crawlers; git pull -v;')
print 'Killing and restarting crawlers...'
with settings(warn_only=True):
run('pkill -KILL -f run_spider_proc;')
run('screen -x; python run_spider_proc.py -r')

def purge_all():
""" Flush all state - on Redis and Mongo and cleanup JSONs and kill crawlers """

for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Syncing code on',host,'...'
run('cd ~/yegii-crawler/crawlers; git pull -v;')
print 'Clearing memory on redis and mongo'
with settings(warn_only=True):
run('cd ~/yegii-crawler/crawlers; redis-cli flushall; mongo yegii drop.js; cd jsons; mkdir -p done2; mv *.json done2;')
print('Killing crawlers ...')
run('pkill -KILL -f run_spider_proc;')

def flush_state():
""" Flush all state - on Redis and Mongo """

for host in crawler_nodes:
env.host_string = host
env.user = 'ubuntu'
print 'Clearing memory on redis and mongo'
with settings(warn_only=True):
run('cd ~/yegii-crawler/crawlers; redis-cli flushall; mongo yegii drop.js;')

Próximo

Anterior

Fabric