Overview of Web Scraping with Python 3.5
January 10, 2017
C
S
C
E
5
9
0
W
e
b
S
c
r
a
p
i
n
g
T
T
o
o
p
p
i
i
c
c
s
s
O
v
e
r
v
i
e
w
–
W
e
b
s
c
r
a
p
i
n
g
I
n
t
r
o
d
u
c
t
i
o
n
t
o
P
y
t
h
o
n
3
.
5
R
R
e
e
a
a
d
d
i
i
n
n
g
g
s
s
:
:
C
h
a
p
t
e
r
1
P
y
t
h
o
n
t
u
t
o
r
i
a
l
C
o
u
r
s
e
I
n
f
o
r
m
a
t
i
o
n
C
o
n
t
a
c
t
I
n
f
o
r
m
a
t
i
o
n
:
•
I
n
s
t
r
u
c
t
o
r
:
M
a
n
t
o
n
M
a
t
t
h
e
w
s
•
O
f
f
i
c
e
:
S
w
e
a
r
i
n
g
e
n
3
A
5
3
•
E
m
a
i
l
:
m
m
<
a
t
>
s
c
•
O
f
f
i
c
e
p
h
o
n
e
:
7
7
7
-
3
2
8
5
•
C
o
u
r
s
e
l
o
c
a
t
i
o
n
:
S
u
m
w
a
l
t
3
0
5
•
C
o
u
r
s
e
T
i
m
e
:
T
R
8
:
3
0
-
9
:
4
5
•
O
f
f
i
c
e
H
o
u
r
s
:
T
R
1
1
:
3
0
-
1
:
0
0
P
M
,
o
t
h
e
r
s
b
y
a
p
p
o
i
n
t
m
e
n
t
T
e
x
t
b
o
o
k
:
R
e
q
u
i
r
e
d
:
W
e
b
S
c
r
a
p
i
n
g
w
i
t
h
P
y
t
h
o
n
–
C
o
l
l
e
c
t
i
n
g
D
a
t
a
f
r
o
m
t
h
e
M
o
d
e
r
n
W
e
b
b
y
R
y
a
n
M
i
t
c
h
e
l
l
,
O
’
R
e
i
l
l
y
,
2
0
1
5
.
R
e
s
o
u
r
c
e
s
-
W
e
b
s
i
t
e
s
o
r
o
n
l
i
n
e
t
e
x
t
s
P
y
t
h
o
n
3
.
5
d
o
c
u
m
e
n
t
a
t
i
o
n
-
h
t
t
p
s
:
/
/
d
o
c
s
.
p
y
t
h
o
n
.
o
r
g
/
3
.
5
/
T
u
t
o
r
i
a
l
(
R
e
q
u
i
r
e
d
)
h
t
t
p
s
:
/
/
d
o
c
s
.
p
y
t
h
o
n
.
o
r
g
/
3
.
5
/
t
u
t
o
r
i
a
l
/
i
n
d
e
x
.
h
t
m
l
T
h
e
S
t
a
n
d
a
r
d
L
i
b
r
a
r
y
h
t
t
p
s
:
/
/
d
o
c
s
.
p
y
t
h
o
n
.
o
r
g
/
3
.
5
/
l
i
b
r
a
r
y
/
i
n
d
e
x
.
h
t
m
l
P
y
t
h
o
n
–
F
l
u
e
n
t
P
y
t
h
o
n
–
C
l
e
a
r
,
C
o
n
c
i
s
e
a
n
d
E
f
f
e
c
t
i
v
e
P
r
o
g
r
a
m
m
i
n
g
b
y
L
u
c
i
a
n
o
R
a
m
a
l
h
o
,
O
’
R
e
i
l
l
y
2
0
1
5
.
N
a
t
u
r
a
l
L
a
n
g
u
a
g
e
T
o
o
l
k
i
t
–
f
o
r
P
y
t
h
o
n
3
.
x
h
t
t
p
:
/
/
w
w
w
.
n
l
t
k
.
o
r
g
/
b
o
o
k
/
a
n
d
f
o
r
P
y
t
h
o
n
2
.
x
h
t
t
p
:
/
/
w
w
w
.
n
l
t
k
.
o
r
g
/
b
o
o
k
_
1
e
d
/
S
c
r
a
p
y
-
h
t
t
p
s
:
/
/
d
o
c
.
s
c
r
a
p
y
.
o
r
g
/
e
n
/
l
a
t
e
s
t
/
i
n
t
r
o
/
t
u
t
o
r
i
a
l
.
h
t
m
l
D
D
i
i
v
v
e
e
i
i
n
n
t
t
o
o
P
P
y
y
t
t
h
h
o
o
n
n
3
3
-
-
h
h
t
t
t
t
p
p
s
s
:
:
/
/
/
/
c
c
l
l
o
o
u
u
d
d
.
.
g
g
i
i
t
t
h
h
u
u
b
b
.
.
c
c
o
o
m
m
/
/
d
d
o
o
w
w
n
n
l
l
o
o
a
a
d
d
s
s
/
/
d
d
i
i
v
v
e
e
i
i
n
n
t
t
o
o
m
m
a
a
r
r
k
k
/
/
d
d
i
i
v
v
e
e
i
i
n
n
t
t
o
o
p
p
y
y
t
t
h
h
o
o
n
n
3
3
/
/
d
d
i
i
v
v
e
e
-
-
i
i
n
n
t
t
o
o
-
-
p
p
y
y
t
t
h
h
o
o
n
n
3
3
.
.
p
p
d
d
f
f
W
h
y
S
c
r
a
p
e
?
“
C
h
e
a
p
e
s
t
f
l
i
g
h
t
t
o
B
o
s
t
o
n
”
G
o
o
g
l
e
k
n
o
w
s
w
h
a
t
a
r
e
o
n
t
h
e
c
o
n
t
e
n
t
p
a
g
e
s
b
u
t
n
o
t
t
h
e
r
e
s
u
l
t
s
o
f
q
u
e
r
i
e
s
a
b
o
u
t
s
p
e
c
i
f
i
c
f
l
i
g
h
t
s
.
A
s
c
r
a
p
e
r
c
a
n
q
u
e
r
y
a
l
l
t
h
e
p
o
p
u
l
a
r
s
i
t
e
s
,
w
i
t
h
u
s
e
r
p
r
e
f
e
r
e
n
c
e
s
a
n
d
o
p
t
i
m
i
z
e
t
h
e
r
e
s
u
l
t
s
t
h
e
w
a
y
y
o
u
w
o
u
l
d
l
i
k
e
V
i
e
w
s
o
f
W
e
b
A
p
p
l
i
c
a
t
i
o
n
s
2
.
1
1
0
0
,
0
0
0
F
e
e
t
:
C
l
i
e
n
t
-
S
e
r
v
e
r
A
r
c
h
i
t
e
c
t
u
r
e
2
.
2
5
0
,
0
0
0
F
e
e
t
:
C
o
m
m
u
n
i
c
a
t
i
o
n
—
H
T
T
P
a
n
d
U
R
I
s
2
.
3
1
0
,
0
0
0
F
e
e
t
:
R
e
p
r
e
s
e
n
t
a
t
i
o
n
—
H
T
M
L
a
n
d
C
S
S
2
.
4
5
,
0
0
0
F
e
e
t
:
3
-
T
i
e
r
A
r
c
h
i
t
e
c
t
u
r
e
&
H
o
r
i
z
o
n
t
a
l
S
c
a
l
i
n
g
2
.
5
1
,
0
0
0
F
e
e
t
:
M
o
d
e
l
-
V
i
e
w
-
C
o
n
t
r
o
l
l
e
r
A
r
c
h
i
t
e
c
t
u
r
e
F
o
x
,
A
r
m
a
n
d
o
;
P
a
t
t
e
r
s
o
n
,
D
a
v
i
d
.
E
n
g
i
n
e
e
r
i
n
g
S
o
f
t
w
a
r
e
a
s
a
S
e
r
v
i
c
e
:
A
n
A
g
i
l
e
A
p
p
r
o
a
c
h
U
s
i
n
g
C
l
o
u
d
C
o
m
p
u
t
i
n
g
(
K
i
n
d
l
e
L
o
c
a
t
i
o
n
s
1
4
8
1
-
1
4
8
5
)
.
S
t
r
a
w
b
e
r
r
y
C
a
n
y
o
n
L
L
C
.
K
i
n
d
l
e
E
d
i
t
i
o
n
.
2
.
1
1
0
0
,
0
0
0
F
e
e
t
V
i
e
w
:
C
l
i
e
n
t
-
S
e
r
v
e
r
A
r
c
h
i
t
e
c
t
u
r
e
H
T
T
P
&
H
T
T
P
/
2
/
/
f
o
r
b
r
o
w
s
e
r
s
s
e
r
v
e
r
s
T
C
P
p
r
o
t
o
c
o
l
I
P
p
r
o
t
o
c
o
l
E
t
h
e
r
n
e
t
p
r
o
t
o
c
o
l
2
.
2
5
0
,
0
0
0
F
e
e
t
:
C
o
m
m
u
n
i
c
a
t
i
o
n
—
H
T
T
P
a
n
d
U
R
I
s
2
.
3
1
0
,
0
0
0
F
e
e
t
:
R
e
p
r
e
s
e
n
t
a
t
i
o
n
—
H
T
M
L
a
n
d
C
S
S
2
.
5
1
,
0
0
0
F
e
e
t
:
M
o
d
e
l
-
V
i
e
w
-
C
o
n
t
r
o
l
l
e
r
A
r
c
h
i
t
e
c
t
u
r
e
I
n
t
r
o
d
u
c
t
i
o
n
t
o
P
y
t
h
o
n
P
y
t
h
o
n
2
.
8
v
s
P
y
t
h
o
n
3
.
5
I
n
s
t
a
l
l
i
n
g
P
y
t
h
o
n
R
e
f
e
r
e
n
c
e
s
P
y
t
h
o
n
3
.
5
d
o
c
u
m
e
n
t
a
t
i
o
n
-
h
t
t
p
s
:
/
/
d
o
c
s
.
p
y
t
h
o
n
.
o
r
g
/
3
.
5
/
T
u
t
o
r
i
a
l
(
R
e
q
u
i
r
e
d
)
h
t
t
p
s
:
/
/
d
o
c
s
.
p
y
t
h
o
n
.
o
r
g
/
3
.
5
/
t
u
t
o
r
i
a
l
/
i
n
d
e
x
.
h
t
m
l
T
h
e
S
t
a
n
d
a
r
d
L
i
b
r
a
r
y
h
t
t
p
s
:
/
/
d
o
c
s
.
p
y
t
h
o
n
.
o
r
g
/
3
.
5
/
l
i
b
r
a
r
y
/
i
n
d
e
x
.
h
t
m
l
D
D
i
i
v
v
e
e
i
i
n
n
t
t
o
o
P
P
y
y
t
t
h
h
o
o
n
n
3
3
-
-
h
h
t
t
t
t
p
p
s
s
:
:
/
/
/
/
c
c
l
l
o
o
u
u
d
d
.
.
g
g
i
i
t
t
h
h
u
u
b
b
.
.
c
c
o
o
m
m
/
/
d
d
o
o
w
w
n
n
l
l
o
o
a
a
d
d
s
s
/
/
d
d
i
i
v
v
e
e
i
i
n
n
t
t
o
o
m
m
a
a
r
r
k
k
/
/
d
d
i
i
v
v
e
e
i
i
n
n
t
t
o
o
p
p
y
y
t
t
h
h
o
o
n
n
3
3
/
/
d
d
i
i
v
v
e
e
-
-
i
i
n
n
t
t
o
o
-
-
p
p
y
y
t
t
h
h
o
o
n
n
3
3
.
.
p
p
d
d
f
f
P
y
t
h
o
n
–
F
l
u
e
n
t
P
y
t
h
o
n
–
C
l
e
a
r
,
C
o
n
c
i
s
e
a
n
d
E
f
f
e
c
t
i
v
e
P
r
o
g
r
a
m
m
i
n
g
b
y
L
u
c
i
a
n
o
R
a
m
a
l
h
o
,
O
’
R
e
i
l
l
y
2
0
1
5
.
P
y
t
h
o
n
i
n
t
e
r
p
r
e
t
e
r
:
E
x
p
r
e
s
s
i
o
n
s
5
0
-
5
*
6
(
5
0
-
5
*
6
)
/
4
8
/
5
#
d
i
v
i
s
i
o
n
a
l
w
a
y
s
r
e
t
u
r
n
s
a
f
l
o
a
t
i
n
g
p
o
i
n
t
n
u
m
b
e
r
1
7
/
3
#
c
l
a
s
s
i
c
d
i
v
i
s
i
o
n
r
e
t
u
r
n
s
a
f
l
o
a
t
1
7
/
/
3
#
f
l
o
o
r
d
i
v
i
s
i
o
n
d
i
s
c
a
r
d
s
t
h
e
f
r
a
c
t
i
o
n
a
l
p
a
r
t
1
7
%
3
#
t
h
e
%
o
p
e
r
a
t
o
r
r
e
t
u
r
n
s
t
h
e
r
e
m
a
i
n
d
e
r
o
f
t
h
e
d
i
v
i
s
i
o
n
5
*
*
2
#
5
s
q
u
a
r
e
d
2
*
*
7
#
2
t
o
t
h
e
p
o
w
e
r
o
f
7
V
a
r
i
a
b
l
e
s
a
n
d
t
y
p
i
n
g
w
i
d
t
h
=
2
0
h
e
i
g
h
t
=
5
*
9
w
i
d
t
h
*
h
e
i
g
h
t
p
r
i
n
t
(
w
i
d
t
h
*
h
e
i
g
h
t
)
#
3
.
1
.
2
S
t
r
i
n
g
s
#
'
s
p
a
m
e
g
g
s
'
#
s
i
n
g
l
e
q
u
o
t
e
s
'
d
o
e
s
n
\
'
t
'
#
u
s
e
\
'
t
o
e
s
c
a
p
e
t
h
e
s
i
n
g
l
e
q
u
o
t
e
.
.
.
"
d
o
e
s
n
'
t
"
#
.
.
.
o
r
u
s
e
d
o
u
b
l
e
q
u
o
t
e
s
i
n
s
t
e
a
d
'
"
Y
e
s
,
"
h
e
s
a
i
d
.
'
"
\
"
Y
e
s
,
\
"
h
e
s
a
i
d
.
"
'
"
I
s
n
\
'
t
,
"
s
h
e
s
a
i
d
.
'
'
"
I
s
n
\
'
t
,
"
s
h
e
s
a
i
d
.
'
p
r
i
n
t
(
'
"
I
s
n
\
'
t
,
"
s
h
e
s
a
i
d
.
'
)
s
=
'
F
i
r
s
t
l
i
n
e
.
\
n
S
e
c
o
n
d
l
i
n
e
.
'
#
\
n
m
e
a
n
s
n
e
w
l
i
n
e
s
#
w
i
t
h
o
u
t
p
r
i
n
t
(
)
,
\
n
i
s
i
n
c
l
u
d
e
d
i
n
t
h
e
o
u
t
p
u
t
p
r
i
n
t
(
s
)
#
w
i
t
h
p
r
i
n
t
(
)
,
\
n
p
r
o
d
u
c
e
s
a
n
e
w
l
i
n
e
p
r
i
n
t
(
'
C
:
\
s
o
m
e
\
n
a
m
e
'
)
#
h
e
r
e
\
n
m
e
a
n
s
n
e
w
l
i
n
e
!
p
r
i
n
t
(
r
'
C
:
\
s
o
m
e
\
n
a
m
e
'
)
#
n
o
t
e
t
h
e
r
b
e
f
o
r
e
t
h
e
q
u
o
t
e
c
o
n
c
a
t
e
n
a
t
i
o
n
#
3
t
i
m
e
s
'
u
n
'
,
f
o
l
l
o
w
e
d
b
y
'
i
u
m
'
#
3
*
'
u
n
'
+
'
i
u
m
'
#
'
P
y
'
'
t
h
o
n
'
p
r
e
f
i
x
=
'
P
y
'
p
r
e
f
i
x
'
t
h
o
n
'
#
c
a
n
'
t
c
o
n
c
a
t
e
n
a
t
e
a
v
a
r
i
a
b
l
e
a
n
d
a
s
t
r
i
n
g
l
i
t
e
r
a
l
w
o
r
d
=
'
P
y
t
h
o
n
'
w
o
r
d
[
0
]
#
c
h
a
r
a
c
t
e
r
i
n
p
o
s
i
t
i
o
n
0
w
o
r
d
[
5
]
#
c
h
a
r
a
c
t
e
r
i
n
p
o
s
i
t
i
o
n
5
w
o
r
d
[
-
1
]
#
l
a
s
t
c
h
a
r
a
c
t
e
r
w
o
r
d
[
-
2
]
#
s
e
c
o
n
d
-
l
a
s
t
c
h
a
r
a
c
t
e
r
w
o
r
d
[
-
6
]
S
l
i
c
e
s
w
o
r
d
[
0
:
2
]
#
c
h
a
r
a
c
t
e
r
s
f
r
o
m
p
o
s
i
t
i
o
n
0
(
i
n
c
l
u
d
e
d
)
t
o
2
(
e
x
c
l
u
d
e
d
)
w
o
r
d
[
2
:
5
]
#
c
h
a
r
a
c
t
e
r
s
f
r
o
m
p
o
s
i
t
i
o
n
2
(
i
n
c
l
u
d
e
d
)
t
o
5
(
e
x
c
l
u
d
e
d
)
w
o
r
d
[
:
2
]
+
w
o
r
d
[
2
:
]
w
o
r
d
[
:
4
]
+
w
o
r
d
[
4
:
]
w
o
r
d
[
:
2
]
#
c
h
a
r
a
c
t
e
r
f
r
o
m
t
h
e
b
e
g
i
n
n
i
n
g
t
o
p
o
s
i
t
i
o
n
2
(
e
x
c
l
u
d
e
d
)
w
o
r
d
[
4
:
]
#
c
h
a
r
a
c
t
e
r
s
f
r
o
m
p
o
s
i
t
i
o
n
4
(
i
n
c
l
u
d
e
d
)
t
o
t
h
e
e
n
d
w
o
r
d
[
-
2
:
]
#
c
h
a
r
a
c
t
e
r
s
f
r
o
m
t
h
e
s
e
c
o
n
d
-
l
a
s
t
(
i
n
c
l
u
d
e
d
)
t
o
t
h
e
e
n
d
w
o
r
d
[
4
2
]
#
t
h
e
w
o
r
d
o
n
l
y
h
a
s
6
c
h
a
r
a
c
t
e
r
s
w
o
r
d
[
4
:
4
2
]
w
o
r
d
[
4
2
:
]
w
o
r
d
[
0
]
=
'
J
'
w
o
r
d
[
2
:
]
=
'
p
y
'
'
J
'
+
w
o
r
d
[
1
:
]
w
o
r
d
[
:
2
]
+
'
p
y
'
s
=
'
s
u
p
e
r
c
a
l
i
f
r
a
g
i
l
i
s
t
i
c
e
x
p
i
a
l
i
d
o
c
i
o
u
s
'
l
e
n
(
s
)
s
q
u
a
r
e
s
=
[
1
,
4
,
9
,
1
6
,
2
5
]
s
a
u
a
r
e
s
s
q
u
a
r
e
s
[
0
]
#
i
n
d
e
x
i
n
g
r
e
t
u
r
n
s
t
h
e
i
t
e
m
s
q
u
a
r
e
s
[
-
1
]
s
q
u
a
r
e
s
[
-
3
:
]
#
s
l
i
c
i
n
g
r
e
t
u
r
n
s
a
n
e
w
l
i
s
t
s
q
u
a
r
e
s
+
[
3
6
,
4
9
,
6
4
,
8
1
,
1
0
0
]
c
u
b
e
s
=
[
1
,
8
,
2
7
,
6
5
,
1
2
5
]
#
s
o
m
e
t
h
i
n
g
'
s
w
r
o
n
g
h
e
r
e
4
*
*
3
#
t
h
e
c
u
b
e
o
f
4
i
s
6
4
,
n
o
t
6
5
!
c
u
b
e
s
[
3
]
=
6
4
#
r
e
p
l
a
c
e
t
h
e
w
r
o
n
g
v
a
l
u
e
c
u
b
e
s
c
u
b
e
s
.
a
p
p
e
n
d
(
2
1
6
)
#
a
d
d
t
h
e
c
u
b
e
o
f
6
c
u
b
e
s
.
a
p
p
e
n
d
(
7
*
*
3
)
#
a
n
d
t
h
e
c
u
b
e
o
f
7
c
u
b
e
s
l
e
t
t
e
r
s
=
[
'
a
'
,
'
b
'
,
'
c
'
,
'
d
'
,
'
e
'
,
'
f
'
,
'
g
'
]
l
e
t
t
e
r
s
#
r
e
p
l
a
c
e
s
o
m
e
v
a
l
u
e
s
l
e
t
t
e
r
s
[
2
:
5
]
=
[
'
C
'
,
'
D
'
,
'
E
'
]
l
e
t
t
e
r
s
#
n
o
w
r
e
m
o
v
e
t
h
e
m
l
e
t
t
e
r
s
[
2
:
5
]
=
[
]
l
e
t
t
e
r
s
#
c
l
e
a
r
t
h
e
l
i
s
t
b
y
r
e
p
l
a
c
i
n
g
a
l
l
t
h
e
e
l
e
m
e
n
t
s
w
i
t
h
a
n
e
m
p
t
y
l
i
s
t
l
e
t
t
e
r
s
[
:
]
=
[
]
l
e
t
t
e
r
s
l
e
t
t
e
r
s
=
[
'
a
'
,
'
b
'
,
'
c
'
,
'
d
'
]
l
e
n
(
l
e
t
t
e
r
s
)
N
e
s
t
i
n
g
a
=
[
'
a
'
,
'
b
'
,
'
c
'
]
n
=
[
1
,
2
,
3
]
x
=
[
a
,
n
]
x
x
[
0
]
x
[
0
]
[
1
]
#
3
.
2
.
F
i
r
s
t
S
t
e
p
s
T
o
w
a
r
d
s
P
r
o
g
r
a
m
m
i
n
g
F
i
b
o
n
a
c
c
i
s
e
r
i
e
s
:
#
t
h
e
s
u
m
o
f
t
w
o
e
l
e
m
e
n
t
s
d
e
f
i
n
e
s
t
h
e
n
e
x
t
a
,
b
=
0
,
1
w
h
i
l
e
b
<
1
0
:
p
r
i
n
t
(
b
)
a
,
b
=
b
,
a
+
b
a
,
b
=
0
,
1
w
h
i
l
e
b
<
1
0
0
0
:
p
r
i
n
t
(
b
,
e
n
d
=
'
,
'
)
a
,
b
=
b
,
a
+
b
i
f
S
t
a
t
e
m
e
n
t
s
x
=
i
n
t
(
i
n
p
u
t
(
"
P
l
e
a
s
e
e
n
t
e
r
a
n
i
n
t
e
g
e
r
:
"
)
)
i
f
x
<
0
:
x
=
0
p
r
i
n
t
(
'
N
e
g
a
t
i
v
e
c
h
a
n
g
e
d
t
o
z
e
r
o
'
)
e
l
i
f
x
=
=
0
:
p
r
i
n
t
(
'
Z
e
r
o
'
)
e
l
i
f
x
=
=
1
:
p
r
i
n
t
(
'
S
i
n
g
l
e
'
)
e
l
s
e
:
p
r
i
n
t
(
'
M
o
r
e
'
)
#
M
e
a
s
u
r
e
s
o
m
e
s
t
r
i
n
g
s
:
w
o
r
d
s
=
[
'
c
a
t
'
,
'
w
i
n
d
o
w
'
,
'
d
e
f
e
n
e
s
t
r
a
t
e
'
]
f
o
r
w
i
n
w
o
r
d
s
:
p
r
i
n
t
(
w
,
l
e
n
(
w
)
)
f
o
r
w
i
n
w
o
r
d
s
[
:
]
:
#
L
o
o
p
o
v
e
r
a
s
l
i
c
e
c
o
p
y
o
f
t
h
e
e
n
t
i
r
e
l
i
s
t
.
i
f
l
e
n
(
w
)
>
6
:
w
o
r
d
s
.
i
n
s
e
r
t
(
0
,
w
)
#
4
.
3
.
T
h
e
r
a
n
g
e
(
)
F
u
n
c
t
i
o
n
f
o
r
i
i
n
r
a
n
g
e
(
5
)
:
p
r
i
n
t
(
i
)
r
a
n
g
e
(
5
,
1
0
)
5
t
h
r
o
u
g
h
9
r
a
n
g
e
(
0
,
1
0
,
3
)
0
,
3
,
6
,
9
r
a
n
g
e
(
-
1
0
,
-
1
0
0
,
-
3
0
)
-
1
0
,
-
4
0
,
-
7
0
a
=
[
'
M
a
r
y
'
,
'
h
a
d
'
,
'
a
'
,
'
l
i
t
t
l
e
'
,
'
l
a
m
b
'
]
f
o
r
i
i
n
r
a
n
g
e
(
l
e
n
(
a
)
)
:
p
r
i
n
t
(
i
,
a
[
i
]
)
p
r
i
n
t
(
r
a
n
g
e
(
1
0
)
)
l
i
s
t
(
r
a
n
g
e
(
5
)
)
B
r
e
a
k
a
n
d
c
o
n
t
i
n
u
e
f
o
r
n
i
n
r
a
n
g
e
(
2
,
1
0
)
:
f
o
r
x
i
n
r
a
n
g
e
(
2
,
n
)
:
i
f
n
%
x
=
=
0
:
p
r
i
n
t
(
n
,
'
e
q
u
a
l
s
'
,
x
,
'
*
'
,
n
/
/
x
)
b
r
e
a
k
e
l
s
e
:
#
l
o
o
p
f
e
l
l
t
h
r
o
u
g
h
w
i
t
h
o
u
t
f
i
n
d
i
n
g
a
f
a
c
t
o
r
p
r
i
n
t
(
n
,
'
i
s
a
p
r
i
m
e
n
u
m
b
e
r
'
)
P
a
s
s
w
h
i
l
e
T
r
u
e
:
p
a
s
s
#
B
u
s
y
-
w
a
i
t
f
o
r
k
e
y
b
o
a
r
d
i
n
t
e
r
r
u
p
t
(
C
t
r
l
+
C
)
c
l
a
s
s
M
y
E
m
p
t
y
C
l
a
s
s
:
p
a
s
s
d
e
f
i
n
i
t
l
o
g
(
*
a
r
g
s
)
:
p
a
s
s
#
R
e
m
e
m
b
e
r
t
o
i
m
p
l
e
m
e
n
t
t
h
i
s
!
#
4
.
6
.
D
e
f
i
n
i
n
g
F
u
n
c
t
i
o
n
s
d
e
f
f
i
b
(
n
)
:
#
w
r
i
t
e
F
i
b
o
n
a
c
c
i
s
e
r
i
e
s
u
p
t
o
n
"
"
"
P
r
i
n
t
a
F
i
b
o
n
a
c
c
i
s
e
r
i
e
s
u
p
t
o
n
.
"
"
"
a
,
b
=
0
,
1
w
h
i
l
e
a
<
n
:
p
r
i
n
t
(
a
,
e
n
d
=
'
'
)
a
,
b
=
b
,
a
+
b
p
r
i
n
t
(
)
This overview delves into the fundamentals of web scraping using Python 3.5, covering topics such as client-server architecture, HTTP communication, URI, HTML, CSS, and more. The course provides insights on why scraping is essential and how it can be used to extract data efficiently. Resources for further learning and recommended readings are also included in the content.
Download Presentation
Please find below an Image/Link to download the presentation.
The content on the website is provided AS IS for your information and personal use only. It may not be sold, licensed, or shared on other websites without obtaining consent from the author. Download presentation by click this link. If you encounter any issues during the download, it is possible that the publisher has removed the file from their server.
E N D
Presentation Transcript
CSCE 590 Web Scraping Topics Overview Web scraping Introduction to Python 3.5 Readings: Chapter 1 Python tutorial January 10, 2017
Course Information Contact Information: Instructor: Manton Matthews Office: Swearingen 3A53 Email: mm <at> sc Office phone: 777-3285 Course location: Sumwalt 305 Course Time: TR 8:30-9:45 Office Hours: TR 11:30-1:00PM, others by appointment Textbook: Required: Web Scraping with Python Collecting Data from the Modern Web by Ryan Mitchell, O Reilly, 2015. 2 CSCE 590 Web Scraping Spring 2017
Resources - Websites or online texts Python 3.5 documentation - https://docs.python.org/3.5/ Tutorial (Required) https://docs.python.org/3.5/tutorial/index.html The Standard Library https://docs.python.org/3.5/library/index.html Python Fluent Python Clear, Concise and Effective Programming by Luciano Ramalho, O Reilly 2015. Natural Language Toolkit for Python 3.x http://www.nltk.org/book/ and for Python2.x http://www.nltk.org/book_1ed/ Scrapy - https://doc.scrapy.org/en/latest/intro/tutorial.html Dive into Python 3- https://cloud.github.com/downloads/diveintomark/dive intopython3/dive-into-python3.pdf 3 CSCE 590 Web Scraping Spring 2017
Why Scrape? Cheapest flight to Boston Google knows what are on the content pages but not the results of queries about specific flights. A scraper can query all the popular sites, with user preferences and optimize the results the way you would like 4 CSCE 590 Web Scraping Spring 2017
Views of Web Applications 2.1 100,000 Feet: Client-Server Architecture 2.2 50,000 Feet: Communication HTTP and URIs 2.3 10,000 Feet: Representation HTML and CSS 2.4 5,000 Feet: 3-Tier Architecture & Horizontal Scaling 2.5 1,000 Feet: Model-View-Controller Architecture Fox, Armando; Patterson, David. Engineering Software as a Service: An Agile Approach Using Cloud Computing (Kindle Locations 1481-1485). Strawberry Canyon LLC. Kindle Edition. 5 CSCE 590 Web Scraping Spring 2017
2.1 100,000 Feet View: Client-Server Architecture HTTP & HTTP/2 // for browsers servers TCP protocol IP protocol Ethernet protocol 6 CSCE 590 Web Scraping Spring 2017
2.2 50,000 Feet: Communication HTTP and URIs 7 CSCE 590 Web Scraping Spring 2017
2.3 10,000 Feet: Representation HTML and CSS 8 CSCE 590 Web Scraping Spring 2017
2.5 1,000 Feet: Model-View- Controller Architecture 9 CSCE 590 Web Scraping Spring 2017
Introduction to Python Python 2.8 vs Python 3.5 10 CSCE 590 Web Scraping Spring 2017
Installing 11 CSCE 590 Web Scraping Spring 2017
Python References Python 3.5 documentation - https://docs.python.org/3.5/ Tutorial (Required) https://docs.python.org/3.5/tutorial/index.html The Standard Library https://docs.python.org/3.5/library/index.html Dive into Python 3- https://cloud.github.com/downloads/diveintomar k/diveintopython3/dive-into-python3.pdf Python Fluent Python Clear, Concise and Effective Programming by Luciano Ramalho, O Reilly 2015. 12 CSCE 590 Web Scraping Spring 2017
Python interpreter: Expressions 50 - 5*6 (50 - 5*6) / 4 8 / 5 # division always returns a floating point number 17 / 3 # classic division returns a float 17 // 3 # floor division discards the fractional part 17 % 3 # the % operator returns the remainder of the division 5 ** 2 # 5 squared 2 ** 7 # 2 to the power of 7 13 CSCE 590 Web Scraping Spring 2017
Variables and typing width = 20 height = 5 * 9 width * height print(width * height) 14 CSCE 590 Web Scraping Spring 2017
# 3.1.2 Strings #'spam eggs' # single quotes 'doesn\'t' # use \' to escape the single quote... "doesn't" # ...or use double quotes instead '"Yes," he said.' "\"Yes,\" he said." '"Isn\'t," she said.' '"Isn\'t," she said.' print('"Isn\'t," she said.') 15 CSCE 590 Web Scraping Spring 2017
s = 'First line.\nSecond line.' # \n means newline s # without print(), \n is included in the output print(s) # with print(), \n produces a new line print('C:\some\name') # here \n means newline! print(r'C:\some\name') # note the r before the quote 16 CSCE 590 Web Scraping Spring 2017
concatenation # 3 times 'un', followed by 'ium' # 3 * 'un' + 'ium' # 'Py' 'thon' prefix = 'Py' prefix 'thon' # can't concatenate a variable and a string literal 17 CSCE 590 Web Scraping Spring 2017
word = 'Python' word[0] # character in position 0 word[5] # character in position 5 word[-1] # last character word[-2] # second-last character word[-6] 18 CSCE 590 Web Scraping Spring 2017
Slices word[0:2] # characters from position 0 (included) to 2 (excluded) word[2:5] # characters from position 2 (included) to 5 (excluded) word[:2] + word[2:] word[:4] + word[4:] word[:2] # character from the beginning to position 2 (excluded) word[4:] # characters from position 4 (included) to the end word[-2:] # characters from the second-last (included) to the end 19 CSCE 590 Web Scraping Spring 2017
word[42] # the word only has 6 characters word[4:42] word[42:] word[0] = 'J' word[2:] = 'py' 'J' + word[1:] word[:2] + 'py' s = 'supercalifragilisticexpialidocious' len(s) 20 CSCE 590 Web Scraping Spring 2017
squares = [1, 4, 9, 16, 25] sauares squares[0] # indexing returns the item squares[-1] squares[-3:] # slicing returns a new list squares + [36, 49, 64, 81, 100] cubes = [1, 8, 27, 65, 125] # something's wrong here 4 ** 3 # the cube of 4 is 64, not 65! cubes[3] = 64 # replace the wrong value cubes cubes.append(216) # add the cube of 6 cubes.append(7 ** 3) # and the cube of 7 cubes 21 CSCE 590 Web Scraping Spring 2017
letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] letters # replace some values letters[2:5] = ['C', 'D', 'E'] letters # now remove them letters[2:5] = [] letters # clear the list by replacing all the elements with an empty list letters[:] = [] letters letters = ['a', 'b', 'c', 'd'] len(letters) 22 CSCE 590 Web Scraping Spring 2017
Nesting a = ['a', 'b', 'c'] n = [1, 2, 3] x = [a, n] x x[0] x[0][1] 23 CSCE 590 Web Scraping Spring 2017
# 3.2. First Steps Towards Programming Fibonacci series: # the sum of two elements defines the next a, b = 0, 1 while b < 10: print(b) a, b = b, a+b 24 CSCE 590 Web Scraping Spring 2017
a, b = 0, 1 while b < 1000: print(b, end=',') a, b = b, a+b 25 CSCE 590 Web Scraping Spring 2017
if Statements x = int(input("Please enter an integer: ")) if x < 0: x = 0 print('Negative changed to zero') elif x == 0: print('Zero') elif x == 1: print('Single') else: print('More') 26 CSCE 590 Web Scraping Spring 2017
# Measure some strings: words = ['cat', 'window', 'defenestrate'] for w in words: print(w, len(w)) for w in words[:]: # Loop over a slice copy of the entire list. if len(w) > 6: words.insert(0, w) 27 CSCE 590 Web Scraping Spring 2017
# 4.3. The range() Function for i in range(5): print(i) range(5, 10) 5 through 9 range(0, 10, 3) 0, 3, 6, 9 range(-10, -100, -30) -10, -40, -70 28 CSCE 590 Web Scraping Spring 2017
a = ['Mary', 'had', 'a', 'little', 'lamb'] for i in range(len(a)): print(i, a[i]) print(range(10)) list(range(5)) 29 CSCE 590 Web Scraping Spring 2017
Break and continue for n in range(2, 10): for x in range(2, n): if n % x == 0: print(n, 'equals', x, '*', n//x) break else: # loop fell through without finding a factor print(n, 'is a prime number') 30 CSCE 590 Web Scraping Spring 2017
Pass while True: pass # Busy-wait for keyboard interrupt (Ctrl+C) class MyEmptyClass: pass def initlog(*args): pass # Remember to implement this! 31 CSCE 590 Web Scraping Spring 2017
# 4.6. Defining Functions def fib(n): # write Fibonacci series up to n """Print a Fibonacci series up to n.""" a, b = 0, 1 while a < n: print(a, end=' ') a, b = b, a+b print() 32 CSCE 590 Web Scraping Spring 2017
