《python数据分析（第2版）-阿曼多.凡丹戈》读书笔记第12章-性能优化、性能分析与并发性及本书附录

 1 import random
 2 from nltk import data
 3 data.path.append(r‘D:\download\nltk_data‘) # 这里的路径需要换成自己数据文件下载的路径
 4 from nltk.corpus import movie_reviews
 5 from nltk.corpus import stopwords
 6 from nltk import FreqDist
 7 from nltk import NaiveBayesClassifier
 8 from nltk.classify import accuracy
 9 
10 import builtins
11 
12 try:
13     profile = builtins.profile
14 except AttributeError:
15     def profile(func): 
16         return func
17 
18 @profile
19 def label_docs():
20     docs = [(list(movie_reviews.words(fid)), cat)
21             for cat in movie_reviews.categories()
22             for fid in movie_reviews.fileids(cat)]
23     random.seed(42)
24     random.shuffle(docs)
25 
26     return docs
27 
28 @profile
29 def isStopWord(word):
30     return word in sw or len(word) == 1
31 
32 @profile
33 def filter_corpus():
34     review_words = movie_reviews.words()
35     print("# Review Words", len(review_words))
36     res = [w.lower() for w in review_words if not isStopWord(w.lower())]
37     print("# After filter", len(res))
38 
39     return res
40 @profile
41 def select_word_features(corpus):
42     words = FreqDist(corpus)
43     N = int(.02 * len(words.keys()))
44     return list(words.keys())[:N]
45 
46 @profile
47 def doc_features(doc):
48     doc_words = FreqDist(w for w in doc if not isStopWord(w))
49     features = {}
50     for word in word_features:
51         features[‘count (%s)‘ % word] = (doc_words.get(word, 0))
52     return features
53 
54 @profile
55 def make_features(docs):
56     return [(doc_features(d), c) for (d,c) in docs]
57 
58 @profile
59 def split_data(sets):
60     return sets[200:], sets[:200]
61 
62 if __name__ == "__main__":
63     labeled_docs = label_docs()
64 
65     sw = set(stopwords.words(‘english‘))
66     filtered = filter_corpus()
67     word_features = select_word_features(filtered)
68     featuresets = make_features(labeled_docs)
69     train_set, test_set = split_data(featuresets)
70     classifier = NaiveBayesClassifier.train(train_set)
71     print("Accuracy", accuracy(classifier, test_set))
72     print(classifier.show_most_informative_features())

1 $ time python D:\Java2018\PythonDataAnalysis2\Chapter12\prof_demo.py

$ python -m cProfile -o D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof D:\Java2018\PythonDataAnalysis2\Chapter12\prof_demo.py

1 $ pip3 install gprof2dot

1 $ gprof2dot -f pstats D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof |dot -Tpng -o D:\Java2018\PythonDataAnalysis2\Chapter12\log\cprof.png

1  python3 -m pstats D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof

/* 
Welcome to the profile statistics browser.
D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof%
D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof% strip
D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof% sort time
D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof% stats 10
Mon Apr  6 20:56:25 2020    D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof

         34307447 function calls (33773715 primitive calls) in 15.954 seconds

   Ordered by: internal time
   List reduced from 4179 to 10 due to restriction <10>

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
   319968    2.242    0.000    2.242    0.000 {method ‘findall‘ of ‘re.Pattern‘ objects}
  2839856    1.118    0.000    1.118    0.000 probability.py:127(__setitem__)
        1    0.944    0.944    1.708    1.708 naivebayes.py:193(train)
   319960    0.915    0.000    1.966    0.000 data.py:1192(readline)
     8006    0.776    0.000    0.776    0.000 {built-in method io.open}
        1    0.720    0.720    3.475    3.475 prof_demo.py:36(<listcomp>)
  6343280    0.642    0.000    7.965    0.000 util.py:271(iterate_from)
     2000    0.616    0.000    1.845    0.001 prof_demo.py:46(doc_features)
  3167640    0.616    0.000    0.718    0.000 prof_demo.py:28(isStopWord)
  3167642    0.475    0.000    4.506    0.000 util.py:408(iterate_from)
 */

D:\Java2018\PythonDataAnalysis2\Chapter12\log\stat.prof% quit
Goodbye.

1 $ pip3 install line_profiler
2 $ kernprof -l -v D:\Java2018\PythonDataAnalysis2\Chapter12\prof_demo.py

Function: label_docs at line 9 Total time: 6.19904 s
Function: isStopWord at line 19 Total time: 2.16542 s
File: prof_demo.py Function: filter_corpus at line 23
Function: select_word_features at line 32 Total time: 4.05266 s
Function: doc_features at line 38 Total time: 12.5919 s
Function: make_features at line 46 Total time: 14.566 s
Function: split_data at line 50 Total time: 3.6e-05 s

1 $ pip3 install cython cytoolz

1 def time(code, n):
2     times = min(timeit.Timer(code, setup=setup).repeat(3, n))
3 
4     return round(1000* np.array(times)/n, 3)

 1 import timeit
 2 import numpy as np
 3 
 4 setup = ‘‘‘
 5 import nltk
 6 import cython_module as cm
 7 import collections
 8 from nltk.corpus import stopwords
 9 from nltk.corpus import movie_reviews
10 from nltk.corpus import names
11 import string
12 import pandas as pd
13 import cytoolz
14 
15 sw = set(stopwords.words(‘english‘))
16 punctuation = set(string.punctuation)
17 all_names = set([name.lower() for name in names.words()])
18 txt = movie_reviews.words(movie_reviews.fileids()[0])
19 
20 def isStopWord(w):
21     return w in sw or w in punctuation
22 
23 def isStopWord2(w):
24     return w in sw or w in punctuation or not w.isalpha()
25 
26 def isStopWord3(w):
27     return w in sw or len(w) == 1 or not w.isalpha() or w in all_names
28 
29 def isStopWord4(w):
30     return w in sw or len(w) == 1
31 
32 def freq_dict(words):
33     dd = collections.defaultdict(int)
34 
35     for word in words:
36         dd[word] += 1
37 
38     return dd
39 
40 def zero_init():
41     features = {}
42 
43     for word in set(txt):
44         features[‘count (%s)‘ % word] = (0)
45 
46 def zero_init2():
47     features = {}
48     for word in set(txt):
49         features[word] = (0)
50 
51 keys = list(set(txt))
52 
53 def zero_init3():
54     features = dict.fromkeys(keys, 0)
55 
56 zero_dict = dict.fromkeys(keys, 0)
57 
58 def dict_copy():
59     features = zero_dict.copy()
60 ‘‘‘
61 
62 def time(code, n):
63     times = min(timeit.Timer(code, setup=setup).repeat(3, n))
64 
65     return round(1000* np.array(times)/n, 3)
66 
67 if __name__ == ‘__main__‘:
68     print("Best of 3 times per loop in milliseconds")
69     n = 10
70     print("zero_init ", time("zero_init()", n))
71     print("zero_init2", time("zero_init2()", n))
72     print("zero_init3", time("zero_init3()", n))
73     print("dict_copy ", time("dict_copy()", n))
74     print("\n")
75 
76     n = 10**2
77     print("isStopWord ", time(‘[w.lower() for w in txt if not isStopWord(w.lower())]‘, n))
78     print("isStopWord2", time(‘[w.lower() for w in txt if not isStopWord2(w.lower())]‘, n))
79     print("isStopWord3", time(‘[w.lower() for w in txt if not isStopWord3(w.lower())]‘, n))
80     print("isStopWord4", time(‘[w.lower() for w in txt if not isStopWord4(w.lower())]‘, n))
81     print("Cythonized isStopWord", time(‘[w.lower() for w in txt if not cm.isStopWord(w.lower())]‘, n))
82     print("Cythonized filter_sw()", time(‘cm.filter_sw(txt)‘, n))
83     print("\n")
84 
85     print("FreqDist", time("nltk.FreqDist(txt)", n))
86     print("Default dict", time(‘freq_dict(txt)‘, n))
87     print("Counter", time(‘collections.Counter(txt)‘, n))
88     print("Series", time(‘pd.Series(txt).value_counts()‘, n))
89     print("Cytoolz", time(‘cytoolz.frequencies(txt)‘, n))
90     print("Cythonized freq_dict", time(‘cm.freq_dict(txt)‘, n))

isStopWord 0.843
isStopWord2 0.902
isStopWord3 0.963
isStopWord4 0.869
Cythonized isStopWord 0.924
Cythonized filter_sw() 0.887

Zero_init 0.61
Zero_init2 0.555
Zero_init3 0.017
Dict_copy 0.011

FreqDist 2.206
Default dict 0.674
Counter 0.79
Series 7.006
Cytoolz 0.542
Cythonized freq_dict 0.616

1 from libc.string cimport strlen

 1 from collections import defaultdict
 2 from nltk.corpus import stopwords
 3 from nltk.corpus import names
 4 from libc.string cimport strlen
 5 from nltk import data
 6 data.path.append(r‘D:\download\nltk_data‘) # 这里的路径需要换成自己数据文件下载的路径
 7 
 8 sw = set(stopwords.words(‘english‘))
 9 all_names = set([name.lower() for name in names.words()])
10 
11 def isStopWord(w):
12      py_byte_string = w.encode(‘UTF-8‘)
13      cdef char* c_string = py_byte_string
14      truth = (w in sw) or (w in all_names) or (not w.isalpha()) or (strlen(c_string) == 1)
15      return truth
16 
17 def filter_sw(words):
18     return [w.lower() for w in words if not isStopWord(w.lower())]
19 
20 def freq_dict(words):
21     dd = defaultdict(int)
22 
23     for word in words:
24         dd[word] += 1
25 
26     return dd

1 from distutils.core import setup
2 from Cython.Build import cythonize
3 
4 setup(
5     ext_modules = cythonize("cython_module.pyx")
6 )

1 $ python3 setup.py build_ext –inplace

$python3 setup.py build_ext –inplace
Running build_ext
Building ‘cython_module’ extension
Creating build
Creating build/temp.macosx-10.12-x86_64-3.6
Clang -Wno-unused-result -Wsign-compare -Wunreachable-code -fno-common –
Dynamic -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes –
I/usr/local/include -I/usr/local/opt/openssl/include –
I/usr/local/opt/sqlite/include –
I/usr/local/Cellar/python3/3.6.0/Frameworks/Python.framework/Versions/3.6/i
Nclude/python3.6m -c cython_module.c -o build/temp.macosx-10.12-
X86_64-3.6/cython_module.o
Clang -bundle -undefined dynamic_lookup build/temp.macosx-10.12-
X86_64-3.6/cython_module.o -L/usr/local/lib -L/usr/local/opt/openssl/lib –
L/usr/local/opt/sqlite/lib -o
/Users/armando/gdrive/projects/bitbucket/pubs/2016-pda-e2-
Packt/chapters/ch-12/cython_module.cpython-36m-darwin.so

 1 from nltk.corpus import movie_reviews
 2 from nltk import data
 3 data.path.append(r‘D:\download\nltk_data‘) # 这里的路径需要换成自己数据文件下载的路径
 4 
 5 from nltk import NaiveBayesClassifier
 6 from nltk.classify import accuracy
 7 import cython_module as cm
 8 import cytoolz
 9 from core import label_docs
10 from core import filter_corpus
11 from core import split_data
12 
13 
14 def select_word_features(corpus):
15     words = cytoolz.frequencies(filtered)
16     sorted_words = sorted(words, key=words.get)
17     N = int(.02 * len(sorted_words))
18 
19     return sorted_words[-N:]
20 
21 def match(a, b):
22     return set(a.keys()).intersection(b)
23 
24 def doc_features(doc):
25     doc_words = cytoolz.frequencies(cm.filter_sw(doc))
26 
27     # initialize to 0
28     features = zero_features.copy()
29 
30     word_matches = match(doc_words, word_features)
31 
32     for word in word_matches:
33         features[word] = (doc_words[word])
34 
35     return features
36 
37 def make_features(docs):
38     return [(doc_features(d), c) for (d,c) in docs]
39 
40 if __name__ == "__main__":
41     labeled_docs = label_docs()
42 
43     filtered = filter_corpus()
44     word_features = select_word_features(filtered)
45     zero_features = dict.fromkeys(word_features, 0)
46     featuresets = make_features(labeled_docs)
47     train_set, test_set = split_data(featuresets)
48     classifier = NaiveBayesClassifier.train(train_set)
49     print("Accuracy", accuracy(classifier, test_set))
50     print(classifier.show_most_informative_features())

1 $ time python3 cython_demo.py

1 p = mp.Pool(nprocs)

1    p.map(simulate, [i for i in range(10, 50)])

 1 from numpy.random import random_integers
 2 from numpy.random import randn
 3 import numpy as np
 4 import timeit
 5 import argparse
 6 import multiprocessing as mp
 7 import matplotlib.pyplot as plt
 8 
 9 
10 def simulate(size):
11     n = 0
12     mean = 0
13     M2 = 0
14 
15     speed = randn(10000)
16 
17     for i in range(1000): 
18         n = n + 1
19         indices = random_integers(0, len(speed)-1, size=size)
20         x = (1 + speed[indices]).prod()
21         delta = x - mean
22         mean = mean + delta/n
23         M2 = M2 + delta*(x - mean)
24 
25     return mean
26 
27 def serial():
28     start = timeit.default_timer()
29 
30     for i in range(10, 50):
31         simulate(i)
32     
33     end = timeit.default_timer() - start
34     print("Serial time", end)
35 
36     return end
37 
38 def parallel(nprocs):
39     start = timeit.default_timer()
40     p = mp.Pool(nprocs)
41     print(nprocs, "Pool creation time", timeit.default_timer() - start)
42 
43     p.map(simulate, [i for i in range(10, 50)])
44     p.close()
45     p.join()
46 
47     end = timeit.default_timer() - start
48     print(nprocs, "Parallel time", end)
49     return end
50 
51 if __name__ == "__main__":
52     ratios = []
53     baseline = serial()
54 
55     for i in range(1, mp.cpu_count()):
56         ratios.append(baseline/parallel(i))
57 
58     plt.xlabel(‘# processes‘)
59     plt.ylabel(‘Serial/Parallel‘)
60     n = np.arange(1, mp.cpu_count())
61     plt.plot(n, ratios)
62     plt.grid(True)
63     plt.show()

1 $ pip3 install joblib

1 def parallel(nprocs):
2     start = timeit.default_timer()
3     Parallel(nprocs)(delayed(simulate)(i) for i in range(10, 50))
4 
5     end = timeit.default_timer() - start
6     print(nprocs, "Parallel time", end)
7     return end

1 $ pip3 install bottleneck

 1 import bottleneck as bn
 2 import numpy as np
 3 import timeit
 4 
 5 
 6 setup = ‘‘‘
 7 import numpy as np
 8 import bottleneck as bn
 9 from scipy.stats import rankdata
10 
11 np.random.seed(42)
12 a = np.random.randn(30)
13 ‘‘‘
14 def time(code, setup, n):
15     return timeit.Timer(code, setup=setup).repeat(3, n)
16 
17 if __name__ == ‘__main__‘:
18     n = 10**3
19     print(n, "pass", max(time("pass", "", n)))
20     print(n, "min np.median", min(time(‘np.median(a)‘, setup, n)))
21     print(n, "min bn.median", min(time(‘bn.median(a)‘, setup, n)))
22     a = np.arange(7)
23     print("Median diff", np.median(a) - bn.median(a))
24     
25     print(n, "min scipy.stats.rankdata", min(time(‘rankdata(a)‘, setup, n)))
26     print(n, "min bn.rankdata", min(time(‘bn.rankdata(a)‘, setup, n)))

1000 pass 8.700000000000374e-06
1000 min np.median 0.03629350000000109
1000 min bn.median 0.0003940999999993977
Median diff 0.0
1000 min scipy.stats.rankdata 0.04359799999999936
1000 min bn.rankdata 0.0010945000000006644

1 $ pip3 install jug

 1 import jug.mapreduce
 2 from jug.compound import CompoundTask
 3 import cython_module as cm
 4 import cytoolz
 5 import pickle
 6 
 7 def get_txts():
 8     return [(1, ‘Lorem ipsum dolor sit amet, consectetur adipiscing elit.‘), 
(2, ‘Donec a elit pharetra, malesuada massa vitae, elementum dolor.‘),
 (3, ‘Integer a tortor ac mi vehicula tempor at a nunc.‘)]
 9 
10 def freq_dict(file_words):
11     filtered = cm.filter_sw(file_words[1].split())
12 
13     fd = cytoolz.frequencies(filtered)
14 
15     return fd
16 
17 def merge(left, right):
18     return cytoolz.merge_with(sum, left, right)
19 
20 merged_counts = CompoundTask(jug.mapreduce.mapreduce, merge, freq_dict, get_txts(), map_step=1)

1 $ jug execute jug_demo.py –jugdir=redis://127.0.0.1/&

1 $ jug status jug_demo.py

1 $ jug cleanup jug_demo.py

1 jug.init(‘jug_demo.py’, ‘redis://127.0.0.1/’)
2 Import jug_demo

1 words = jug.task.value(jug_demo.merged_counts)

1 import jug
2 
3 def main():
4     jug.init(‘jug_demo.py‘, ‘redis://127.0.0.1/‘)
5     import jug_demo
6     print("Merged counts", jug.task.value(jug_demo.merged_counts))
7 
8 if __name__ == "__main__":
9     main()

1 $ ./configure –prefix=/usr/local
2 $ make all
3 $ sudo make install

1 $ pip3 install mpi4py

$ export LC_ALL=en_US.UTF-8
$ export LANG=en_US.UTF-8

1 $ ipython3 profile create –parallel –profile=mpi

1 $ ipcluster start –profile=mpi –engines=MPI –debug

1 $ jupyter-notebook –profile=mpi –log-level=DEBUG

1 from ipyparallel import Client
2 import statsmodels.api as sm

1 data_loader = sm.datasets.sunspots.load_pandas()
2 vals = data_loader.data[‘SUNACTIVITY’].values
3 glob_mean = vals.mean()
4 glob_mean

Out [2]: 49.752103559870541

1 c = Client(profile=’mpi’)

1 view=c[:]

1 view.activate()

 1 from mpi4py import MPI
 2 from numpy.random import random_integers
 3 from numpy.random import randn
 4 import numpy as np
 5 import statsmodels.api as sm
 6 import bottleneck as bn
 7 import logging
 8 
 9 
10 def jackknife(a, parallel=True):
11     data_loader = sm.datasets.sunspots.load_pandas()
12     vals = data_loader.data[‘SUNACTIVITY‘].values
13 
14     results = []
15 
16     for i in a:
17         tmp = np.array(vals.tolist())
18         tmp[i] = np.nan
19         results.append(bn.nanmean(tmp))
20 
21     results = np.array(results)
22 
23     if parallel:
24         comm = MPI.COMM_WORLD
25         rcvBuf = np.zeros(results.shape)
26         comm.gather([results, MPI.DOUBLE], [rcvBuf, MPI.DOUBLE])
27 
28     return results
29 
30 if __name__ == "__main__":
31     skiplist = np.arange(39, dtype=‘int‘)
32     print(jackknife(skiplist, True))
33     

1 view.run(‘mpi_ipython.py‘)

1 view.scatter(‘a‘,np.arange(len(vals),dtype=‘int‘))

1 view[‘a‘]

Out[8]:[array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
33, 34, 35, 36, 37, 38]), … TRUNCATED …]

1 %px means = jackknife(a)

1 view[‘means‘]

1 all_means = []
2 for v in view[‘means‘]:
3     all_means.extend(v)
4     mean(all_means)

Out [11]: 49.752103559870577

1 hist(all_means, bins=sqrt(len(all_means)))

 [(k, c.metadata[k][‘started‘], c.metadata[k][‘pyout‘],
c.metadata[k][‘pyerr‘]) for k in c.metadata.keys()]