Skip to content
Snippets Groups Projects
Commit d7b6b570 authored by Felipe Ramos's avatar Felipe Ramos
Browse files

solving version conflict

parent a1ff06c4
No related branches found
No related tags found
No related merge requests found
<OGRWFSDataSource> <OGRWFSDataSource>
<URL>http://openapi.aurin.org.au/wfs?version=1.0.0&TYPENAME=</URL> <URL>http://openapi.aurin.org.au/wfs?version=1.0.0&TYPENAME aurin:datasource-AU_Govt_ABS_Census-UoM_AURIN_DB_2_gccsa_p11b_english_profic_by_arrival_yr_by_sex_census_2016</URL>
<HttpAuth>BASIC</HttpAuth> <HttpAuth>BASIC</HttpAuth>
......
import configparser
import urllib.request
from lxml import etree
username='student'
password='dj78dfGF'
def openapi_request(url):
# create an authenticated HTTP handler and submit URL
password_manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, url, username, password)
auth_manager = urllib.request.HTTPBasicAuthHandler(password_manager)
opener = urllib.request.build_opener(auth_manager)
urllib.request.install_opener(opener)
req = urllib.request.Request(url)
handler = urllib.request.urlopen(req)
return handler.read()
dataset = 'aurin:datasource-AU_Govt_ABS_Census-UoM_AURIN_DB_2_gccsa_p11b_english_profic_by_arrival_yr_by_sex_census_2016'
url ='http://openapi.aurin.org.au/wfs?request=DescribeFeatureType&service=WFS&version=1.1.0&typeName='+dataset
xml = openapi_request(url)
root = etree.fromstring(xml)
import xmltodict, json
out = xmltodict.parse(xml)
\ No newline at end of file
def save2Couch(twt, db): def save2Couch(twt, db, w_replace = False):
doc = twt doc = twt
doc['_id'] = doc['id_str'] doc['_id'] = doc['id_str']
try: try:
db.save(doc) db.save(doc)
except: except:
if w_replace:
replaceCDB(doc, db) replaceCDB(doc, db)
......
...@@ -12,6 +12,12 @@ from preprocess import save2couch, updateCBD, replaceCDB, classifyTwt ...@@ -12,6 +12,12 @@ from preprocess import save2couch, updateCBD, replaceCDB, classifyTwt
collector_type = sys.argv[1] collector_type = sys.argv[1]
region = sys.argv[2] region = sys.argv[2]
couchIP = sys.argv[3]
#"172.26.130.79"
storeCouch = True
couchserver = couchdb.Server("http://admin:admin@"+couchIP+":5984/")
db = couchserver['live_demo2']
#change code to your current Twitter account #change code to your current Twitter account
...@@ -44,15 +50,13 @@ class StreamListener(tweepy.StreamListener): ...@@ -44,15 +50,13 @@ class StreamListener(tweepy.StreamListener):
for char in [',','\n']: for char in [',','\n']:
text.replace(char, " ") text.replace(char, " ")
#sports = ['afl', 'tennis', 'footie','swimming','AustralianOpen', 'soccer', 'cricket', '#afl']
#if any(s in text.lower() for s in sports): #if any(s in text.lower() for s in sports):
if not is_retweet: if not is_retweet:
doc = status._json doc = status._json
doc['region'] = self.config['region'] doc['region'] = self.config['region']
doc['text'] = text doc['text'] = text
#save2Couch(doc, self.db) #s
#self.file.append(json.load(doc)) #self.file.append(json.load(doc))
serial = {doc['id_str']:doc} serial = {doc['id_str']:doc}
...@@ -60,7 +64,9 @@ class StreamListener(tweepy.StreamListener): ...@@ -60,7 +64,9 @@ class StreamListener(tweepy.StreamListener):
# print(doc['text'], doc['user']['location']) # print(doc['text'], doc['user']['location'])
#else: #else:
print(doc['text']) print(doc['text'])
if storeCouch:
ave2Couch(doc, self.db)
else:
outfile.write(str(serial) + '\n') outfile.write(str(serial) + '\n')
def on_error(self, status_code): def on_error(self, status_code):
...@@ -78,9 +84,6 @@ if __name__ == "__main__": ...@@ -78,9 +84,6 @@ if __name__ == "__main__":
config = sets[int(region)] config = sets[int(region)]
#couchserver = couchdb.Server("http://admin:admin@172.26.130.79:5984/")
#db = couchserver['twitter_demo']
API_key = config['secret']['API_key'] API_key = config['secret']['API_key']
API_secret = config['secret']['API_secret'] API_secret = config['secret']['API_secret']
access_token = config['secret']['access_token'] access_token = config['secret']['access_token']
...@@ -90,8 +93,6 @@ if __name__ == "__main__": ...@@ -90,8 +93,6 @@ if __name__ == "__main__":
auth = tweepy.OAuthHandler(API_key, API_secret) auth = tweepy.OAuthHandler(API_key, API_secret)
auth.set_access_token(access_token, access_secret) auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth,wait_on_rate_limit=True, wait_on_rate_limit_notify=True) api = tweepy.API(auth,wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
#outfile = open('twt_stream.json', 'r+', encoding ='utf-8')
if collector_type == 'stream': if collector_type == 'stream':
...@@ -109,7 +110,6 @@ if __name__ == "__main__": ...@@ -109,7 +110,6 @@ if __name__ == "__main__":
continue continue
elif collector_type == 'search': elif collector_type == 'search':
cityCoord = config['point'] cityCoord = config['point']
...@@ -176,19 +176,19 @@ if __name__ == "__main__": ...@@ -176,19 +176,19 @@ if __name__ == "__main__":
ids_.append(tweet.id_str) ids_.append(tweet.id_str)
final[tweet.id_str] = tweet final[tweet.id_str] = tweet
#save results to disk #save results to disk
serializable = {k: v._json for k,v in final.items()} serializable = {k: v._json for k,v in final.items()}
for k, v in serializable.items(): for k, v in serializable.items():
v['region'] = config['region'] v['region'] = config['region']
#v['tags'] = classifyTwt(v) v['tags'] = classifyTwt(v)
# Save to couch dv
#save2Couch(v) if storeCouch:
save2Couch(v)
else:
#Save to disk as json. Check if saving file exist and increase # version
current_version = '-1' current_version = '-1'
versions = [] versions = []
for filename in os.listdir('./'): for filename in os.listdir('./'):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment