import numpy as np
import nltk
import matplotlib
import matplotlib.pyplot as plt
import wordcloud
plt.style.use('fivethirtyeight')
%matplotlib inline

# Read in corpus from local corpus directory
corpus = nltk.corpus.PlaintextCorpusReader("corpus", '.*\.txt')

# Calculate basic statistics
works = corpus.fileids()
titles = [title.split()[0] for title in works]
print("Total works by Danielle Steele: {}".format(len(works)))

word_count = [len(corpus.words(work)) for work in works]
print("Average word count {:.2f}".format(np.average(word_count)))
print("Median word count {:.2f}".format(np.median(word_count)))
print("Standard deviation word count {:.2f}".format(np.std(word_count)))
print("Minimum words {}".format(np.min(word_count)))
print("Maximum words {}".format(np.max(word_count)))

Total works by Danielle Steele: 88
Average word count 133223.85
Median word count 135017.50
Standard deviation word count 34262.29
Minimum words 56394
Maximum words 215792

# View the distribution of works by word count
n, bins, patches = plt.hist(word_count, 20, range=(0,220000))
_ = plt.ylabel("Total works")
_ = plt.xlabel("Total words")

print("{:.1f}%".format(len([words for words in word_count if words < 150000 and words > 100000])/len(works)*100))

54.5%

# Create a wordcloud to look at textual data
wordcloud_im = wordcloud.WordCloud(width=800, height=600)
_ = wordcloud_im.generate(corpus.raw())
_ = plt.imshow(wordcloud_im)
_ = plt.axis("off")

# Extract locations
# After spending some time trying to roll my own from the GNIS dataset and other gazetteers I decided 
# to use the Stanford NER tagger
from nltk.tag.stanford import StanfordNERTagger
# NLTK lib makes calling out to the pretained model (first arg) and classifier (second arg)
# very seamless, requires java.
tagger = StanfordNERTagger("ner/classifiers/english.all.3class.distsim.crf.ser.gz", "ner/stanford-ner.jar")
tagged_corpus = []
for work in corpus.fileids():
    tagged_corpus.append(tagger.tag(corpus.words(work)))

# Extract locations
filtered_locations = []
for tagged_work in tagged_corpus:
    # Add all consecutive tags, if there is a spacing of 3 or more separate
    spacing = 0
    SPACE_LIMIT = 2
    work_storage = []
    location_storage = []
    for word in tagged_work:
        # if it is a location word add it
        if word[1] == "LOCATION":
            location_storage.append(word[0])
        else:
            # if it is not a spacing word, consume a space
            spacing += 1
            # if the spacing limit is reached add the aggregated location
            # and reset the parameters
            if spacing > SPACE_LIMIT:
                # Add only nonempty lists
                if len(location_storage) != 0:
                    work_storage.append(location_storage[:])
                spacing = 0
                location_storage = []
    filtered_locations.append(work_storage)
    work_storage = []

# Utility function for adding all of the tags together
def create_place_name(tagged_locs :list):
    agg_string = ""
    for place_name in tagged_locs:
        agg_string += place_name
        agg_string += " "
    return agg_string

# Take a peak at the filtered locations to get an idea of accuracy
# but first a sanity check
locations_only = [create_place_name(item) for sublist in filtered_locations for item in sublist]
location_works = [] # list of lists, containing locations, matches up with corpus.fileids() order
for work in filtered_locations:
    location_works.append([create_place_name(item) for item in work])
    
print("Do we have 88 works as expected?: {}".format(len(filtered_locations) == 88))
print("Total number of locations tagged: {}".format(len(locations_only)))


print([create_place_name(places) for places in filtered_locations[0][:5]])
print([create_place_name(places) for places in filtered_locations[15][50:55]])
print([create_place_name(places) for places in filtered_locations[50][100:105]])
print([create_place_name(places) for places in filtered_locations[80][155:160]])

Do we have 88 works as expected?: True
Total number of locations tagged: 46230
['Victoria ', 'New York ', 'West Village ', 'Tribeca ', 'West Village ']
['Berlin ', 'Berlin ', 'Verdun ', 'Berlin ', 'Berlin ']
['Santa Claus ', 'New York ', 'California ', 'New York ', 'New York ']
['France ', 'Riviera Nice ', 'London ', 'New York ', 'Rome ']

# Distribution of location counts
_ = plt.hist([len(x) for x in location_works])
_ = plt.ylabel("Total Works")
_ = plt.xlabel("Total Locations Mentioned")

# Can't geocode all so, selecting a random book
# to explore locations
import geopy
import time
geocoder = geopy.GoogleV3(api_key='XXXXXXXXXXXX')
book_locs = location_works[30]
geocoded_locs = []
for loc in book_locs:
    geocoded_locs.append(geocoder.geocode(loc))

# Adapted from example code found http://matplotlib.org/basemap/users/robin.html
lat = [loc.latitude for loc in geocoded_locs if loc != None]
long = [loc.longitude for loc in geocoded_locs if loc != None]
latlong = zip(lat, long)

from mpl_toolkits.basemap import Basemap
m = Basemap(projection='robin',lon_0=-80, resolution='l')
m.drawcoastlines()
m.drawcountries()
m.fillcontinents(color='yellow',lake_color='black')
# draw parallels and meridians.
# m.drawparallels(np.arange(-90.,120.,30.), color='white')
# m.drawmeridians(np.arange(0.,360.,30.), color='white')
m.drawmapboundary(fill_color='black')
x, y = m(long, lat)
m.plot(x, y, 'r.', alpha=0.3, markersize=5)
plt.title("Locations mentioned in {}".format(titles[30]))
plt.show()