Skip to content

Commit 923dae9

Browse files
author
Rakshith Bhyravabhotla
authored
[Storage] Bug fix - Upload from requests get fails in get_length (#8539)
* Bug fix - Upload from requests get fails in get_length * changes * shared common * remove attr error * cahnges * remove recordigns * live test only * fix a test
1 parent a5b5510 commit 923dae9

7 files changed

Lines changed: 430 additions & 3 deletions

File tree

‎sdk/storage/azure-storage-blob/azure/storage/blob/_shared/request_handlers.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,12 @@ def get_length(data):
6666
except (AttributeError, UnsupportedOperation):
6767
pass
6868
else:
69-
return fstat(fileno).st_size
69+
try:
70+
return fstat(fileno).st_size
71+
except OSError:
72+
# Not a valid fileno, may be possible requests returned
73+
# a socket number?
74+
pass
7075

7176
# If the stream is seekable and tell() is implemented, calculate the stream size.
7277
try:

‎sdk/storage/azure-storage-blob/tests/recordings/test_common_blob.test_create_blob_with_generator.yaml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
Accept:
6+
- '*/*'
7+
Accept-Encoding:
8+
- gzip, deflate
9+
Connection:
10+
- keep-alive
11+
Content-Length:
12+
- '0'
13+
User-Agent:
14+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
15+
x-ms-date:
16+
- Fri, 08 Nov 2019 18:26:18 GMT
17+
x-ms-version:
18+
- '2019-02-02'
19+
method: PUT
20+
uri: https://storagename.blob.core.windows.net/utcontainere08213a7?timeout=5&restype=container
21+
response:
22+
body:
23+
string: ''
24+
headers:
25+
content-length:
26+
- '0'
27+
date:
28+
- Fri, 08 Nov 2019 18:26:18 GMT
29+
etag:
30+
- '"0x8D7647925CFA591"'
31+
last-modified:
32+
- Fri, 08 Nov 2019 18:26:18 GMT
33+
server:
34+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
35+
x-ms-version:
36+
- '2019-02-02'
37+
status:
38+
code: 201
39+
message: Created
40+
- request:
41+
body: helloworld! eom
42+
headers:
43+
Accept:
44+
- '*/*'
45+
Accept-Encoding:
46+
- gzip, deflate
47+
Connection:
48+
- keep-alive
49+
Content-Length:
50+
- '15'
51+
Content-Type:
52+
- application/octet-stream
53+
User-Agent:
54+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
55+
x-ms-date:
56+
- Fri, 08 Nov 2019 18:26:19 GMT
57+
x-ms-version:
58+
- '2019-02-02'
59+
method: PUT
60+
uri: https://storagename.blob.core.windows.net/utcontainere08213a7/gen_blob?blockid=TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQSUzRA%3D%3D&comp=block
61+
response:
62+
body:
63+
string: ''
64+
headers:
65+
content-length:
66+
- '0'
67+
date:
68+
- Fri, 08 Nov 2019 18:26:18 GMT
69+
server:
70+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
71+
x-ms-content-crc64:
72+
- 7Vr3DTw/xmg=
73+
x-ms-request-server-encrypted:
74+
- 'true'
75+
x-ms-version:
76+
- '2019-02-02'
77+
status:
78+
code: 201
79+
message: Created
80+
- request:
81+
body: '<?xml version=''1.0'' encoding=''utf-8''?>
82+
83+
<BlockList><Latest>TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQSUzRA==</Latest></BlockList>'
84+
headers:
85+
Accept:
86+
- '*/*'
87+
Accept-Encoding:
88+
- gzip, deflate
89+
Connection:
90+
- keep-alive
91+
Content-Length:
92+
- '143'
93+
Content-Type:
94+
- application/xml; charset=utf-8
95+
If-None-Match:
96+
- '*'
97+
User-Agent:
98+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
99+
x-ms-date:
100+
- Fri, 08 Nov 2019 18:26:19 GMT
101+
x-ms-version:
102+
- '2019-02-02'
103+
method: PUT
104+
uri: https://storagename.blob.core.windows.net/utcontainere08213a7/gen_blob?comp=blocklist
105+
response:
106+
body:
107+
string: ''
108+
headers:
109+
content-length:
110+
- '0'
111+
date:
112+
- Fri, 08 Nov 2019 18:26:18 GMT
113+
etag:
114+
- '"0x8D7647925E1758B"'
115+
last-modified:
116+
- Fri, 08 Nov 2019 18:26:18 GMT
117+
server:
118+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
119+
x-ms-content-crc64:
120+
- 7XagkOrfW/A=
121+
x-ms-request-server-encrypted:
122+
- 'true'
123+
x-ms-version:
124+
- '2019-02-02'
125+
status:
126+
code: 201
127+
message: Created
128+
- request:
129+
body: null
130+
headers:
131+
Accept:
132+
- application/xml
133+
Accept-Encoding:
134+
- gzip, deflate
135+
Connection:
136+
- keep-alive
137+
User-Agent:
138+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
139+
x-ms-date:
140+
- Fri, 08 Nov 2019 18:26:19 GMT
141+
x-ms-range:
142+
- bytes=0-33554431
143+
x-ms-version:
144+
- '2019-02-02'
145+
method: GET
146+
uri: https://storagename.blob.core.windows.net/utcontainere08213a7/gen_blob
147+
response:
148+
body:
149+
string: helloworld! eom
150+
headers:
151+
accept-ranges:
152+
- bytes
153+
content-length:
154+
- '15'
155+
content-range:
156+
- bytes 0-14/15
157+
content-type:
158+
- application/octet-stream
159+
date:
160+
- Fri, 08 Nov 2019 18:26:18 GMT
161+
etag:
162+
- '"0x8D7647925E1758B"'
163+
last-modified:
164+
- Fri, 08 Nov 2019 18:26:18 GMT
165+
server:
166+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
167+
x-ms-blob-type:
168+
- BlockBlob
169+
x-ms-creation-time:
170+
- Fri, 08 Nov 2019 18:26:18 GMT
171+
x-ms-lease-state:
172+
- available
173+
x-ms-lease-status:
174+
- unlocked
175+
x-ms-server-encrypted:
176+
- 'true'
177+
x-ms-version:
178+
- '2019-02-02'
179+
status:
180+
code: 206
181+
message: Partial Content
182+
version: 1

‎sdk/storage/azure-storage-blob/tests/recordings/test_common_blob_async.test_create_blob_with_generator_async.yaml‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
User-Agent:
6+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
7+
x-ms-date:
8+
- Fri, 08 Nov 2019 18:26:19 GMT
9+
x-ms-version:
10+
- '2019-02-02'
11+
method: PUT
12+
uri: https://storagename.blob.core.windows.net/utcontaineree1d18a1?timeout=5&restype=container
13+
response:
14+
body:
15+
string: ''
16+
headers:
17+
content-length: '0'
18+
date: Fri, 08 Nov 2019 18:26:18 GMT
19+
etag: '"0x8D76479260E2832"'
20+
last-modified: Fri, 08 Nov 2019 18:26:18 GMT
21+
server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
22+
x-ms-version: '2019-02-02'
23+
status:
24+
code: 201
25+
message: Created
26+
url: !!python/object/new:yarl.URL
27+
state: !!python/tuple
28+
- !!python/object/new:urllib.parse.SplitResult
29+
- https
30+
- pyacrstoragebs7lblezedhd.blob.core.windows.net
31+
- /utcontaineree1d18a1
32+
- timeout=5&restype=container
33+
- ''
34+
- request:
35+
body: helloworld! eom
36+
headers:
37+
Content-Length:
38+
- '15'
39+
Content-Type:
40+
- application/octet-stream
41+
User-Agent:
42+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
43+
x-ms-date:
44+
- Fri, 08 Nov 2019 18:26:19 GMT
45+
x-ms-version:
46+
- '2019-02-02'
47+
method: PUT
48+
uri: https://storagename.blob.core.windows.net/utcontaineree1d18a1/gen_blob?blockid=TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQSUzRA%3D%3D&comp=block
49+
response:
50+
body:
51+
string: ''
52+
headers:
53+
content-length: '0'
54+
date: Fri, 08 Nov 2019 18:26:18 GMT
55+
server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
56+
x-ms-content-crc64: 7Vr3DTw/xmg=
57+
x-ms-request-server-encrypted: 'true'
58+
x-ms-version: '2019-02-02'
59+
status:
60+
code: 201
61+
message: Created
62+
url: !!python/object/new:yarl.URL
63+
state: !!python/tuple
64+
- !!python/object/new:urllib.parse.SplitResult
65+
- https
66+
- pyacrstoragebs7lblezedhd.blob.core.windows.net
67+
- /utcontaineree1d18a1/gen_blob
68+
- blockid=TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQSUzRA%3D%3D&comp=block
69+
- ''
70+
- request:
71+
body: '<?xml version=''1.0'' encoding=''utf-8''?>
72+
73+
<BlockList><Latest>TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQXdNREF3TURBd01EQSUzRA==</Latest></BlockList>'
74+
headers:
75+
Content-Length:
76+
- '143'
77+
Content-Type:
78+
- application/xml; charset=utf-8
79+
If-None-Match:
80+
- '*'
81+
User-Agent:
82+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
83+
x-ms-date:
84+
- Fri, 08 Nov 2019 18:26:19 GMT
85+
x-ms-version:
86+
- '2019-02-02'
87+
method: PUT
88+
uri: https://storagename.blob.core.windows.net/utcontaineree1d18a1/gen_blob?comp=blocklist
89+
response:
90+
body:
91+
string: ''
92+
headers:
93+
content-length: '0'
94+
date: Fri, 08 Nov 2019 18:26:18 GMT
95+
etag: '"0x8D76479261D022C"'
96+
last-modified: Fri, 08 Nov 2019 18:26:18 GMT
97+
server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
98+
x-ms-content-crc64: 7XagkOrfW/A=
99+
x-ms-request-server-encrypted: 'true'
100+
x-ms-version: '2019-02-02'
101+
status:
102+
code: 201
103+
message: Created
104+
url: !!python/object/new:yarl.URL
105+
state: !!python/tuple
106+
- !!python/object/new:urllib.parse.SplitResult
107+
- https
108+
- pyacrstoragebs7lblezedhd.blob.core.windows.net
109+
- /utcontaineree1d18a1/gen_blob
110+
- comp=blocklist
111+
- ''
112+
- request:
113+
body: null
114+
headers:
115+
Accept:
116+
- application/xml
117+
User-Agent:
118+
- azsdk-python-storage-blob/12.1.0b1 Python/3.7.3 (Windows-10-10.0.17763-SP0)
119+
x-ms-date:
120+
- Fri, 08 Nov 2019 18:26:20 GMT
121+
x-ms-range:
122+
- bytes=0-33554431
123+
x-ms-version:
124+
- '2019-02-02'
125+
method: GET
126+
uri: https://storagename.blob.core.windows.net/utcontaineree1d18a1/gen_blob
127+
response:
128+
body:
129+
string: helloworld! eom
130+
headers:
131+
accept-ranges: bytes
132+
content-length: '15'
133+
content-range: bytes 0-14/15
134+
content-type: application/octet-stream
135+
date: Fri, 08 Nov 2019 18:26:18 GMT
136+
etag: '"0x8D76479261D022C"'
137+
last-modified: Fri, 08 Nov 2019 18:26:18 GMT
138+
server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
139+
x-ms-blob-type: BlockBlob
140+
x-ms-creation-time: Fri, 08 Nov 2019 18:26:18 GMT
141+
x-ms-lease-state: available
142+
x-ms-lease-status: unlocked
143+
x-ms-server-encrypted: 'true'
144+
x-ms-version: '2019-02-02'
145+
status:
146+
code: 206
147+
message: Partial Content
148+
url: !!python/object/new:yarl.URL
149+
state: !!python/tuple
150+
- !!python/object/new:urllib.parse.SplitResult
151+
- https
152+
- pyacrstoragebs7lblezedhd.blob.core.windows.net
153+
- /utcontaineree1d18a1/gen_blob
154+
- ''
155+
- ''
156+
version: 1

‎sdk/storage/azure-storage-blob/tests/test_common_blob.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,35 @@ def test_create_blob_with_lease_id(self, resource_group, location, storage_accou
247247
content = blob.download_blob(lease=lease).readall()
248248
self.assertEqual(content, data)
249249

250+
@GlobalStorageAccountPreparer()
251+
def test_create_blob_with_generator(self, resource_group, location, storage_account, storage_account_key):
252+
self._setup(storage_account.name, storage_account_key)
253+
254+
# Act
255+
def gen():
256+
yield "hello"
257+
yield "world!"
258+
yield " eom"
259+
blob = self.bsc.get_blob_client(self.container_name, "gen_blob")
260+
resp = blob.upload_blob(data=gen())
261+
262+
# Assert
263+
self.assertIsNotNone(resp.get('etag'))
264+
content = blob.download_blob().readall()
265+
self.assertEqual(content, b"helloworld! eom")
266+
267+
@pytest.mark.live_test_only
268+
@GlobalStorageAccountPreparer()
269+
def test_create_blob_with_requests(self, resource_group, location, storage_account, storage_account_key):
270+
self._setup(storage_account.name, storage_account_key)
271+
272+
# Act
273+
uri = "http://www.gutenberg.org/files/59466/59466-0.txt"
274+
data = requests.get(uri, stream=True)
275+
blob = self.bsc.get_blob_client(self.container_name, "gutenberg")
276+
resp = blob.upload_blob(data=data.raw)
277+
278+
self.assertIsNotNone(resp.get('etag'))
250279

251280
@GlobalStorageAccountPreparer()
252281
def test_create_blob_with_metadata(self, resource_group, location, storage_account, storage_account_key):

0 commit comments

Comments
 (0)
⚡