admin管理员组

文章数量:1026373

An API is returning to me this random google bucket url when trying to upload larger files to it:

/...-storage-bucket-prod/google-oauth2%.../...?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=blob-creator-reader%40tctn-eu-services-979d.iam.gserviceaccount%2...%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241116T214909Z&X-Goog-Expires=59&X-Goog-Signature=...X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-hash

I looked at trying to PUT to it, I even tried to give it host, x-goog-content-length-range, x-goog-hash for each chunk as I go through them, but it keeps returning the same error no matter what I do:

host:storage.googleapis
x-goog-content-length-range:0,16383
x-goog-hash:crc32c=ZVBtjQ==

host;x-goog-content-length-range;x-goog-hash
UNSIGNED-PAYLOAD</CanonicalRequest></Error>

My code:

def calculate_crc32c(data: bytes):
    return base64.b64encode(crc32c(data).to_bytes(4, 'big')).decode('utf-8')


def put_file(api: 'API', file: 'File', data: bytes):
    checksum_bs4 = calculate_crc32c(data)
    content_length = len(data)

    position = 0

    def file_chunk_generator(chunk_size=16384):
        nonlocal sync_event, data, content_length, position
        for i in range(0, len(data), chunk_size):
            chunk = data[i:i + chunk_size]
            yield chunk

    with httpx.Client(http2=False) as request:
        response = request.put(
            FILES_URL.format(api.document_storage_uri, file.hash),
            content=file_chunk_generator(),
            headers=(headers := {
                **api.session.headers,
                'content-length': str(content_length),
                'content-type': 'application/octet-stream',
                'x-goog-hash': f'crc32c={checksum_bs4}',
            })
        )

    if response.status_code == 302:
        position = 0
        print("Full google upload detected, continuing")
        url = response.headers.get("Location")
        print(url)
        with httpx.Client(http2=False) as request:

            for chunk in file_chunk_generator():
                print(f'{position},{position + len(chunk) - 1}')
                response = request.put(
                    url,
                    content=chunk,
                    headers={
                        'host': 'storage.googleapis',
                        'x-goog-content-length-range': f'{position},{(position + len(chunk) -1)}',
                        'x-goog-hash': f'crc32c={calculate_crc32c(chunk)}',
                        'Content-Length': str(len(chunk)),
                    }
                )
                if response.status_code != 200:
                    break
                else:
                    print("SUCCESS!!!")

    if response.status_code != 200:
        raise Exception(f"Put file failed - {response.status_code}\n{response.text}")
    else:
        print(file.uuid, "uploaded")

An API is returning to me this random google bucket url when trying to upload larger files to it:

https://storage.googleapis/...-storage-bucket-prod/google-oauth2%.../...?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=blob-creator-reader%40tctn-eu-services-979d.iam.gserviceaccount%2...%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241116T214909Z&X-Goog-Expires=59&X-Goog-Signature=...X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-hash

I looked at trying to PUT to it, I even tried to give it host, x-goog-content-length-range, x-goog-hash for each chunk as I go through them, but it keeps returning the same error no matter what I do:

host:storage.googleapis
x-goog-content-length-range:0,16383
x-goog-hash:crc32c=ZVBtjQ==

host;x-goog-content-length-range;x-goog-hash
UNSIGNED-PAYLOAD</CanonicalRequest></Error>

My code:

def calculate_crc32c(data: bytes):
    return base64.b64encode(crc32c(data).to_bytes(4, 'big')).decode('utf-8')


def put_file(api: 'API', file: 'File', data: bytes):
    checksum_bs4 = calculate_crc32c(data)
    content_length = len(data)

    position = 0

    def file_chunk_generator(chunk_size=16384):
        nonlocal sync_event, data, content_length, position
        for i in range(0, len(data), chunk_size):
            chunk = data[i:i + chunk_size]
            yield chunk

    with httpx.Client(http2=False) as request:
        response = request.put(
            FILES_URL.format(api.document_storage_uri, file.hash),
            content=file_chunk_generator(),
            headers=(headers := {
                **api.session.headers,
                'content-length': str(content_length),
                'content-type': 'application/octet-stream',
                'x-goog-hash': f'crc32c={checksum_bs4}',
            })
        )

    if response.status_code == 302:
        position = 0
        print("Full google upload detected, continuing")
        url = response.headers.get("Location")
        print(url)
        with httpx.Client(http2=False) as request:

            for chunk in file_chunk_generator():
                print(f'{position},{position + len(chunk) - 1}')
                response = request.put(
                    url,
                    content=chunk,
                    headers={
                        'host': 'storage.googleapis',
                        'x-goog-content-length-range': f'{position},{(position + len(chunk) -1)}',
                        'x-goog-hash': f'crc32c={calculate_crc32c(chunk)}',
                        'Content-Length': str(len(chunk)),
                    }
                )
                if response.status_code != 200:
                    break
                else:
                    print("SUCCESS!!!")

    if response.status_code != 200:
        raise Exception(f"Put file failed - {response.status_code}\n{response.text}")
    else:
        print(file.uuid, "uploaded")
  • python
  • google-cloud-storage
Share Improve this question asked Nov 16, 2024 at 21:58 RedTTGRedTTG 547 bronze badges 4
  • It's a Google Signed URL. See using Signed URLs with resumable uploads – DazWilkin Commented Nov 16, 2024 at 22:13
  • @DazWilkin then why is it not happy with my headers? – RedTTG Commented Nov 16, 2024 at 23:23
  • You will want to POST to the Signed URL with Content-Type and Content-Length headers. If the service will accept resumable uploads, you'll be given a different URL to PUT chunks too. – DazWilkin Commented Nov 17, 2024 at 16:32
  • Your question is not a minimal repro and would benefit from including specific details of "the same error" and possibly the API that you're using that's generating the Signed URL. – DazWilkin Commented Nov 17, 2024 at 16:33
Add a comment  | 

1 Answer 1

Reset to default 0

The API was passing x-goog-content-length of 0,7GB that was the required header, not the chunk length

An API is returning to me this random google bucket url when trying to upload larger files to it:

/...-storage-bucket-prod/google-oauth2%.../...?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=blob-creator-reader%40tctn-eu-services-979d.iam.gserviceaccount%2...%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241116T214909Z&X-Goog-Expires=59&X-Goog-Signature=...X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-hash

I looked at trying to PUT to it, I even tried to give it host, x-goog-content-length-range, x-goog-hash for each chunk as I go through them, but it keeps returning the same error no matter what I do:

host:storage.googleapis
x-goog-content-length-range:0,16383
x-goog-hash:crc32c=ZVBtjQ==

host;x-goog-content-length-range;x-goog-hash
UNSIGNED-PAYLOAD</CanonicalRequest></Error>

My code:

def calculate_crc32c(data: bytes):
    return base64.b64encode(crc32c(data).to_bytes(4, 'big')).decode('utf-8')


def put_file(api: 'API', file: 'File', data: bytes):
    checksum_bs4 = calculate_crc32c(data)
    content_length = len(data)

    position = 0

    def file_chunk_generator(chunk_size=16384):
        nonlocal sync_event, data, content_length, position
        for i in range(0, len(data), chunk_size):
            chunk = data[i:i + chunk_size]
            yield chunk

    with httpx.Client(http2=False) as request:
        response = request.put(
            FILES_URL.format(api.document_storage_uri, file.hash),
            content=file_chunk_generator(),
            headers=(headers := {
                **api.session.headers,
                'content-length': str(content_length),
                'content-type': 'application/octet-stream',
                'x-goog-hash': f'crc32c={checksum_bs4}',
            })
        )

    if response.status_code == 302:
        position = 0
        print("Full google upload detected, continuing")
        url = response.headers.get("Location")
        print(url)
        with httpx.Client(http2=False) as request:

            for chunk in file_chunk_generator():
                print(f'{position},{position + len(chunk) - 1}')
                response = request.put(
                    url,
                    content=chunk,
                    headers={
                        'host': 'storage.googleapis',
                        'x-goog-content-length-range': f'{position},{(position + len(chunk) -1)}',
                        'x-goog-hash': f'crc32c={calculate_crc32c(chunk)}',
                        'Content-Length': str(len(chunk)),
                    }
                )
                if response.status_code != 200:
                    break
                else:
                    print("SUCCESS!!!")

    if response.status_code != 200:
        raise Exception(f"Put file failed - {response.status_code}\n{response.text}")
    else:
        print(file.uuid, "uploaded")

An API is returning to me this random google bucket url when trying to upload larger files to it:

https://storage.googleapis/...-storage-bucket-prod/google-oauth2%.../...?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=blob-creator-reader%40tctn-eu-services-979d.iam.gserviceaccount%2...%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241116T214909Z&X-Goog-Expires=59&X-Goog-Signature=...X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-hash

I looked at trying to PUT to it, I even tried to give it host, x-goog-content-length-range, x-goog-hash for each chunk as I go through them, but it keeps returning the same error no matter what I do:

host:storage.googleapis
x-goog-content-length-range:0,16383
x-goog-hash:crc32c=ZVBtjQ==

host;x-goog-content-length-range;x-goog-hash
UNSIGNED-PAYLOAD</CanonicalRequest></Error>

My code:

def calculate_crc32c(data: bytes):
    return base64.b64encode(crc32c(data).to_bytes(4, 'big')).decode('utf-8')


def put_file(api: 'API', file: 'File', data: bytes):
    checksum_bs4 = calculate_crc32c(data)
    content_length = len(data)

    position = 0

    def file_chunk_generator(chunk_size=16384):
        nonlocal sync_event, data, content_length, position
        for i in range(0, len(data), chunk_size):
            chunk = data[i:i + chunk_size]
            yield chunk

    with httpx.Client(http2=False) as request:
        response = request.put(
            FILES_URL.format(api.document_storage_uri, file.hash),
            content=file_chunk_generator(),
            headers=(headers := {
                **api.session.headers,
                'content-length': str(content_length),
                'content-type': 'application/octet-stream',
                'x-goog-hash': f'crc32c={checksum_bs4}',
            })
        )

    if response.status_code == 302:
        position = 0
        print("Full google upload detected, continuing")
        url = response.headers.get("Location")
        print(url)
        with httpx.Client(http2=False) as request:

            for chunk in file_chunk_generator():
                print(f'{position},{position + len(chunk) - 1}')
                response = request.put(
                    url,
                    content=chunk,
                    headers={
                        'host': 'storage.googleapis',
                        'x-goog-content-length-range': f'{position},{(position + len(chunk) -1)}',
                        'x-goog-hash': f'crc32c={calculate_crc32c(chunk)}',
                        'Content-Length': str(len(chunk)),
                    }
                )
                if response.status_code != 200:
                    break
                else:
                    print("SUCCESS!!!")

    if response.status_code != 200:
        raise Exception(f"Put file failed - {response.status_code}\n{response.text}")
    else:
        print(file.uuid, "uploaded")
  • python
  • google-cloud-storage
Share Improve this question asked Nov 16, 2024 at 21:58 RedTTGRedTTG 547 bronze badges 4
  • It's a Google Signed URL. See using Signed URLs with resumable uploads – DazWilkin Commented Nov 16, 2024 at 22:13
  • @DazWilkin then why is it not happy with my headers? – RedTTG Commented Nov 16, 2024 at 23:23
  • You will want to POST to the Signed URL with Content-Type and Content-Length headers. If the service will accept resumable uploads, you'll be given a different URL to PUT chunks too. – DazWilkin Commented Nov 17, 2024 at 16:32
  • Your question is not a minimal repro and would benefit from including specific details of "the same error" and possibly the API that you're using that's generating the Signed URL. – DazWilkin Commented Nov 17, 2024 at 16:33
Add a comment  | 

1 Answer 1

Reset to default 0

The API was passing x-goog-content-length of 0,7GB that was the required header, not the chunk length

本文标签： pythonHow do I upload to this google urlStack Overflow

版权声明：本文标题：python - How do I upload to this google url - Stack Overflow 内容由热心网友自发贡献，该文观点仅代表作者本人， 转载请联系作者并注明出处：http://it.en369.cn/questions/1745642098a2160809.html， 本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌抄袭侵权/违法违规的内容，一经查实，本站将立刻删除。