from fabric.api import run from fabric.api import hosts, local, settings, abort from fabric.state import env import os crawler_nodes = ('yegiic', 'yegiic4', 'yegiic5', 'yegiic6', 'yegiic7', 'yegiic8') def proxy_restart():    """ Pull latest configuration and restart proxy nodes """    for host in ('proxylb','proxylb2', 'proxylb3'):        env.host_string = host        env.user = 'ubuntu'        run('cd ~/proxyrotate; git pull -v; source ~/proxy/bin/activate; python rotate_proxies.py -w; python rotate_proxies.py -W; sudo service haproxy restart') def proxy_monitor_restart():    """ Pull latest configuration and restart proxy nodes """    for host in ('proxylb','proxylb2', 'proxylb3'):        env.host_string = host        env.user = 'ubuntu'        with settings(warn_only=True):                  run('cd ~/proxyrotate; source ~/proxy/bin/activate; pkill -KILL -f proxy_monitor_restart; python proxy_monitor_restart.py')         def git_pull():    for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Syncing code on',host,'...'        run('cd ~/yegii-crawler/crawlers; git pull -v') def kill_all_crawls():    for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Killing all crawls...'        run('pkill -KILL -f run_spider_proc')         def kill_crawl():    with settings(warn_only=True):        env.user = 'ubuntu'        crawl_id = raw_input('Enter crawl id: ').strip()        print 'Killing crawl id',crawl_id        run('cd ~/yegii-crawler/crawlers/; ~/yegi/bin/python mark_done.py %s 1' % crawl_id) def free_buffers():    env.user = 'ubuntu'    run('sudo su -c "echo 3 > /proc/sys/vm/drop_caches"; free -m') def remove_crawl_logs():    for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Dropping crawl logs older than 3 days on',host        run('cd ~/yegii-crawler/crawlers/logs/; find *.log -mtime +3 | xargs rm -f -;')         def apply_firewall():    for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Syncing code on',host,'...'        run('cd ~/yegii-crawler/; cp iptables.rules ~; sudo iptables-restore < iptables.rules')         def restart_crawlers():    for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Syncing code on',host,'...'        run('cd ~/yegii-crawler/crawlers; git pull -v;')        print 'Killing and restarting crawlers...'        with settings(warn_only=True):                          run('pkill -KILL -f run_spider_proc;')                  run('screen -x; python run_spider_proc.py -r') def purge_all():    """ Flush all state - on Redis and Mongo and cleanup JSONs and kill crawlers """        for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Syncing code on',host,'...'        run('cd ~/yegii-crawler/crawlers; git pull -v;')        print 'Clearing memory on redis and mongo'        with settings(warn_only=True):                  run('cd ~/yegii-crawler/crawlers; redis-cli flushall; mongo yegii drop.js; cd jsons; mkdir -p done2; mv *.json done2;')            print('Killing crawlers ...')            run('pkill -KILL -f run_spider_proc;')       def flush_state():    """ Flush all state - on Redis and Mongo """        for host in crawler_nodes:        env.host_string = host        env.user = 'ubuntu'        print 'Clearing memory on redis and mongo'        with settings(warn_only=True):                  run('cd ~/yegii-crawler/crawlers; redis-cli flushall; mongo yegii drop.js;')

car_model_to_brand = {'suzuki swift vxi': 'suzuki',                       'maruti wagon r': 'maruti',                       'honda civic x': 'honda'} car_brand_to_capbrand = {'suzuki': 'suzuKi'} def lookup(car_model):     key = car_model     # remove extra spaces, lowercase, convert - to _     nkey = ' '.join(map(lambda x: x.lower().replace('-','_').strip(), key.split()))     model = car_model_to_brand[nkey]     if model in car_brand_to_capbrand:         return car_brand_to_capbrand[model]     else:         # Follow normal capitalization routine         return " ".join(map(lambda x: x.capitalize().strip(), model.split()))      class A(object):     def __init__(self, fruits=[], idx=0):         self.fruits = fruits[:]         self.idx = idx              def __gt__(self, obj):         return self.idx < obj.idx     def __str__(self):         return self.__class__.__name__ class B(A):     pass class C(A):     pass class ParaSingleton(object):     """ Parametrized Singleton - uses arguments to decide the singleton state """     _instances = {}          def __new__(cls, in_dict):         key = tuple(sorted(in_dict.items()))         if key in cls._instances:             return cls._instances[key]         else:             instance = object.__new__(cls)             cls._instances[key] = instance             return instance class Segment(Singleton):          def __init__(self, in_dict):         self.s1 = in_dict.get('s1')         self.s2 = in_dict.get('s2')         self.s3 = in_dict.get('s3')         self.s4 = in_dict.get('s4')     def __eq__(self, obj):         return self.s1 == obj.s1 and self.s2 == obj.s2 and self.s3 == obj.s3 and self.s4 == obj.s4      def test(fruits):     a = A(fruits)     b = B(fruits,idx=2)     c = C(fruits,idx=1)          c.fruits.append('banana')     b.fruits.append('grape')     print c.fruits     print b.fruits     print a.fruits     objs = [a,b,c]     print objs     print sorted(objs) def test2():     d1 = {'s1': 'a','s2': 'b','s3': 'c', 's4': 'd'}     d2 = {'s1': 'b','s2': 'c','s3': 'd', 's4': 'e'}      s1 = Segment(d1)     s2 = Segment(d1)     s3 = Segment(d1)     sc = Segment(d2)          print s1 == s2     print s1 == sc     print s2 == sc     print s1 is s2     print s1 is s3     print '==========='          set1 = {s1, s2, s3}     set2 = {s2, s3}     print set1.issubset(set2)     # print set2       if __name__ == "__main__":     # fruits=['lemon','mango','orange']     # test(fruits)     test2()