Skip to content

Commit 8697311

Browse files
RafiotRafiot
committed
chg: Add support for storage state
1 parent f2b2591 commit 8697311

File tree

2 files changed

+149
-1
lines changed

2 files changed

+149
-1
lines changed

‎pylacus/api.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ class CaptureResponse(TypedDict, total=False):
3434
last_redirected_url: str | None
3535
har: dict[str, Any] | None
3636
cookies: list[dict[str, str]] | None
37+
storage: dict[str, Any] | None
3738
error: str | None
3839
html: str | None
3940
png: bytes | None
@@ -51,6 +52,7 @@ class CaptureResponseJson(TypedDict, total=False):
5152
last_redirected_url: str | None
5253
har: dict[str, Any] | None
5354
cookies: list[dict[str, str]] | None
55+
storage: dict[str, Any] | None
5456
error: str | None
5557
html: str | None
5658
png: str | None
@@ -73,6 +75,7 @@ class CaptureSettings(TypedDict, total=False):
7375
proxy: str | dict[str, str] | None
7476
general_timeout_in_sec: int | None
7577
cookies: list[dict[str, Any]] | None
78+
storage: dict[str, Any] | None
7679
headers: str | dict[str, str] | None
7780
http_credentials: dict[str, str] | None
7881
geolocation: dict[str, float] | None
@@ -147,6 +150,7 @@ def enqueue(self, *,
147150
proxy: str | dict[str, str] | None=None,
148151
general_timeout_in_sec: int | None=None,
149152
cookies: list[dict[str, Any]] | None=None,
153+
storage: dict[str, Any] | None=None,
150154
headers: str | dict[str, str] | None=None,
151155
http_credentials: dict[str, str] | None=None,
152156
geolocation: dict[str, float] | None=None,
@@ -178,6 +182,7 @@ def enqueue(self, *,
178182
proxy: str | dict[str, str] | None=None,
179183
general_timeout_in_sec: int | None=None,
180184
cookies: list[dict[str, Any]] | None=None,
185+
storage: dict[str, Any] | None=None,
181186
headers: str | dict[str, str] | None=None,
182187
http_credentials: dict[str, str] | None=None,
183188
geolocation: dict[str, float] | None=None,
@@ -224,6 +229,8 @@ def enqueue(self, *,
224229
to_enqueue['general_timeout_in_sec'] = general_timeout_in_sec
225230
if cookies:
226231
to_enqueue['cookies'] = cookies
232+
if storage:
233+
to_enqueue['storage'] = storage
227234
if headers:
228235
to_enqueue['headers'] = headers
229236
if http_credentials:

‎tests/test_web.py‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
#!/usr/bin/env python3
22

3+
import json
34
import time
45
import unittest
56

7+
from datetime import datetime
8+
69
from pylacus import PyLacus
710
from pylacus.api import CaptureStatus
811

@@ -18,11 +21,149 @@ def test_up(self) -> None:
1821
self.assertFalse(self.client.is_busy())
1922

2023
def test_submit(self) -> None:
21-
uuid = self.client.enqueue(url="circl.lu")
24+
uuid = self.client.enqueue(url="circl.lu", max_retries=0)
2225
while True:
2326
status = self.client.get_capture_status(uuid)
2427
if status == CaptureStatus.DONE:
2528
break
2629
time.sleep(5)
2730
response = self.client.get_capture(uuid)
2831
self.assertEqual(response['status'], 1)
32+
33+
def test_submit_cookies(self) -> None:
34+
# Very basic cookie
35+
uuid = self.client.enqueue(url="circl.lu", cookies="{\"test\": \"test\"}", max_retries=0)
36+
while True:
37+
status = self.client.get_capture_status(uuid)
38+
if status == CaptureStatus.DONE:
39+
break
40+
time.sleep(5)
41+
response = self.client.get_capture(uuid)
42+
self.assertEqual(len(response['cookies']), 1)
43+
self.assertEqual(response['cookies'][0]['domain'], 'circl.lu')
44+
self.assertEqual(response['cookies'], response['storage']['cookies'])
45+
46+
cookie_from_capture = response['cookies'][0]
47+
expires = datetime.now().timestamp() + 3600
48+
cookie_from_capture['expires'] = expires
49+
uuid = self.client.enqueue(url="circl.lu", cookies=cookie_from_capture, max_retries=0)
50+
while True:
51+
status = self.client.get_capture_status(uuid)
52+
if status == CaptureStatus.DONE:
53+
break
54+
time.sleep(5)
55+
response = self.client.get_capture(uuid)
56+
self.assertEqual(response['cookies'][0]['expires'], expires)
57+
58+
# Send list
59+
cookies_from_capture = response['cookies']
60+
expires = datetime.now().timestamp() + 4000
61+
cookies_from_capture[0]['expires'] = expires
62+
uuid = self.client.enqueue(url="circl.lu", cookies=cookies_from_capture, max_retries=0)
63+
while True:
64+
status = self.client.get_capture_status(uuid)
65+
if status == CaptureStatus.DONE:
66+
break
67+
time.sleep(5)
68+
response = self.client.get_capture(uuid)
69+
self.assertEqual(response['cookies'][0]['expires'], expires)
70+
71+
# Send list as json
72+
cookies_from_capture = response['cookies']
73+
uuid = self.client.enqueue(url="circl.lu", cookies=json.dumps(cookies_from_capture), max_retries=0)
74+
while True:
75+
status = self.client.get_capture_status(uuid)
76+
if status == CaptureStatus.DONE:
77+
break
78+
time.sleep(5)
79+
response = self.client.get_capture(uuid)
80+
self.assertEqual(response['cookies'][0]['expires'], expires, response['cookies'])
81+
82+
def test_submit_storage(self) -> None:
83+
uuid = self.client.enqueue(url="https://mdn.github.io/dom-examples/web-storage/",
84+
max_retries=0)
85+
while True:
86+
status = self.client.get_capture_status(uuid)
87+
if status == CaptureStatus.DONE:
88+
break
89+
time.sleep(5)
90+
response = self.client.get_capture(uuid)
91+
storage = response['storage']
92+
self.assertEqual(storage['origins'][0]['origin'], 'https://mdn.github.io')
93+
self.assertEqual(len(storage['origins'][0]['localStorage']), 3)
94+
for item in storage['origins'][0]['localStorage']:
95+
if item['name'] == 'font':
96+
self.assertEqual(item['value'], 'sans-serif')
97+
elif item['name'] == 'image':
98+
self.assertEqual(item['value'], 'images/firefoxos.png')
99+
elif item['name'] == 'bgcolor':
100+
self.assertEqual(item['value'], 'FF0000')
101+
102+
# Change values
103+
for item in storage['origins'][0]['localStorage']:
104+
if item['name'] == 'font':
105+
item['value'] = 'serif'
106+
elif item['name'] == 'image':
107+
item['value'] = 'images/crocodile.png'
108+
elif item['name'] == 'bgcolor':
109+
item['value'] = '00FF00'
110+
111+
uuid = self.client.enqueue(url="https://mdn.github.io/dom-examples/web-storage/",
112+
storage=storage,
113+
max_retries=0)
114+
while True:
115+
status = self.client.get_capture_status(uuid)
116+
if status == CaptureStatus.DONE:
117+
break
118+
time.sleep(5)
119+
response = self.client.get_capture(uuid)
120+
storage = response['storage']
121+
self.assertEqual(storage['origins'][0]['origin'], 'https://mdn.github.io')
122+
self.assertEqual(len(storage['origins'][0]['localStorage']), 3)
123+
for item in storage['origins'][0]['localStorage']:
124+
if item['name'] == 'font':
125+
self.assertEqual(item['value'], 'serif')
126+
elif item['name'] == 'image':
127+
self.assertEqual(item['value'], 'images/crocodile.png')
128+
elif item['name'] == 'bgcolor':
129+
self.assertEqual(item['value'], '00FF00')
130+
131+
def test_submit_indexeddb(self) -> None:
132+
uuid = self.client.enqueue(url="https://mdn.github.io/dom-examples/indexeddb-api/index.html",
133+
# headless=False,
134+
headers={'foo': 'bar'},
135+
max_retries=0)
136+
while True:
137+
status = self.client.get_capture_status(uuid)
138+
if status == CaptureStatus.DONE:
139+
break
140+
time.sleep(5)
141+
response = self.client.get_capture(uuid)
142+
storage = response['storage']
143+
self.assertEqual(storage['origins'][0]['origin'], 'https://mdn.github.io')
144+
self.assertEqual(len(storage['origins'][0]['indexedDB']), 1)
145+
indexeddb = storage['origins'][0]['indexedDB'][0]
146+
self.assertEqual(indexeddb['name'], 'mdn-demo-indexeddb-epublications')
147+
self.assertEqual(len(indexeddb['stores'][0]['records']), 0)
148+
149+
# Add record
150+
record = {'value': {'biblioid': 'Bar',
151+
'title': 'Foo',
152+
'year': 1999, 'id': 1}}
153+
storage['origins'][0]['indexedDB'][0]['stores'][0]['records'].append(record)
154+
uuid = self.client.enqueue(url="https://mdn.github.io/dom-examples/indexeddb-api/index.html",
155+
# headless=False,
156+
storage=storage,
157+
max_retries=0)
158+
while True:
159+
status = self.client.get_capture_status(uuid)
160+
if status == CaptureStatus.DONE:
161+
break
162+
time.sleep(5)
163+
response = self.client.get_capture(uuid)
164+
storage = response['storage']
165+
self.assertEqual(storage['origins'][0]['origin'], 'https://mdn.github.io')
166+
self.assertEqual(len(storage['origins'][0]['indexedDB']), 1)
167+
indexeddb = storage['origins'][0]['indexedDB'][0]
168+
self.assertEqual(indexeddb['name'], 'mdn-demo-indexeddb-epublications')
169+
self.assertEqual(len(indexeddb['stores'][0]['records']), 1)

0 commit comments

Comments
 (0)