SkyHub-Crawler-Python/main.py at master · Vegas007/SkyHub-Crawler-Python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
#!/usr/bin/env python
print("Hello World!!")

import sys
from multiprocessing import Process

sys.path.insert(0, 'Crawler')
sys.path.insert(0, 'Crawler/Crawlers')
sys.path.insert(0, 'Crawler/Crawlers/Apps')
sys.path.insert(0, 'Crawler/Helpers')
sys.path.insert(0, 'SmartCrawlers/WayBackMachine')
sys.path.insert(0, 'Server')

def CrawlerWayBackMachine():
    from Crawler.SmartCrawlers.WayBackMachine import CrawlerWayBackMachine
    crawlerWayBackMachine = CrawlerWayBackMachine.CrawlerWayBackMachine()

    from Crawler.Crawlers.Interfaces.Apps.CrawlerPHPBB import CrawlerPHPBB

    #hackpedia.info
    crawlerWayBackMachine.start(CrawlerPHPBB(user="muflonel2000", url="hackpedia.info", forumGrandParentId='1_forum_14996534590908914', websiteName='Hackpedia', websiteImage="https://a.wattpad.com/useravatar/anon_hacker.128.204872.jpg", websiteCover="https://www.cryptus.in/image/online-training-cceh.jpg", websiteCountry="Romania", websiteCity="Bucharest", websiteLanguage="Romanian"))

    #from Crawler.Crawlers.CrawlerTurkishajan import  CrawlerTurkishajan
    #crawlerWayBackMachine.start(CrawlerTurkishajan(user="muflonel2000", url="turkishajan.com", forumGrandParentId='1_forum_14996534590908914', websiteName='Turkishajan', websiteImage="https://pbs.twimg.com/profile_images/660874924815310848/ymph0vVr.jpg", websiteCover="https://www.technobezz.com/files/uploads/2016/01/Turkish-Hacker-Sentenced-To-344-years-1170x644.jpg", websiteCountry="Turkey", websiteCity="Instanbul", websiteLanguage="Turkish"))

    # from Crawler.Crawlers.CrawlerPHPBBForums import CrawlerPHPBBForums
    # crawlerWayBackMachine.start(CrawlerPHPBBForums, "hackpedia.info")


def testServerAPI():
    from Server.ServerAPI import ServerAPI
    print(ServerAPI.getAddress("Ferdinand Street, No 28", "Romania"))

    ServerAPI.loginUser("muflonel2000")
    ServerAPI.loginUser("muflonel2000")
    ServerAPI.loginUser("muflonel2000")
    topic = ServerAPI.postAddTopic("","","muflonel2000","","TITLU TEST77","DESCRIERE","",["misto","coool","awesome"],[],"2017-08-27T13:55:54+00:00","Romania","City","Romanian",-666,-666)
    forum = ServerAPI.postAddForum("","","muflonel2000","","NAME - FORUM","NAME - FORUM TITLE","DESCRIERE","https://cdn4.iconfinder.com/data/icons/ionicons/512/icon-image-128.png","http://colorfully.eu/wp-content/uploads/2012/10/empty-road-highway-with-fog-facebook-cover.jpg",["misto","coool","awesome"],"2017-08-27T13:55:54+00:00","Romania","City","Romanian",-666,-666)
    #reply = ServerAPI.postAddReply("", "admin",topic,"","Reply5","DESCRIERE",["misto","coool","awesome"],[],"2017-08-27T13:55:54+00:00","Romania","City","Romanian",-666,-666)


def testProductPriceCurrency():

    from Crawler.Objects.Products.ObjectProductPrice import ObjectProductPrice
    price = ObjectProductPrice()
    price.testCurrencyConverter()

def run_spider(crawl):
    if __name__ == '__main__':
        p = Process(target=crawl.start_requests)
        p.start()
        p.join()

def CrawlerScrapy():
    from scrapy.crawler import CrawlerProcess

    scrapyProcess = CrawlerProcess({
            'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
        })

    from Crawler.Crawlers.Interfaces.Apps.News.CrawlerAntena3 import CrawlerAntena3
    CrawlerAntena3.MAXIMUM_NUMBER_PAGES = 1000
    CrawlerAntena3.INFINITE_LOOP = True
    scrapyProcess.crawl(CrawlerAntena3)

    from Crawler.Crawlers.Interfaces.Apps.News.CrawlerFonduriUeRo import CrawlerFonduriUeRo
    CrawlerFonduriUeRo.MAXIMUM_NUMBER_PAGES = 1000
    CrawlerFonduriUeRo.INFINITE_LOOP = True
    scrapyProcess.crawl(CrawlerFonduriUeRo)

    from Crawler.Crawlers.Interfaces.Apps.News.CrawlerHotnews import CrawlerHotnews
    CrawlerHotnews.MAXIMUM_NUMBER_PAGES = 1000
    CrawlerHotnews.INFINITE_LOOP = True
    scrapyProcess.crawl(CrawlerHotnews)

    # from Crawler.Crawlers.Interfaces.Apps.Products.CrawlerEbay import  CrawlerEbay
    # scrapyProcess.crawl(CrawlerEbay)

    # from Crawler.Crawlers.Interfaces.Apps.Events.CrawlerNewAmericaOrg import CrawlerNewAmericaOrg
    # scrapyProcess.crawl(CrawlerNewAmericaOrg)
    #
    # from Crawler.Crawlers.Interfaces.Apps.Events.CrawlerCatoOrg import CrawlerCatoOrg
    # scrapyProcess.crawl(CrawlerCatoOrg)

    scrapyProcess.start()  # the script will block here until the crawling is finished


#init main

# testServerAPI()

#CrawlerWayBackMachine()


CrawlerScrapy()

#testProductPriceCurrency()