Page MenuHomePhorge

Has anyone tried using the `conduit` file uploadchunk method?
Closed, ResolvedPublic

Asked by dmoxyeze on Jun 7 2023, 08:29.

Details

I am trying to use the file uploadchunk method to upload a file in chunks because of size constraint, but the documentation doesn't say how much on how to achieve this.

Answers

This answer has been hidden.

avivey
Updated 407 Days Ago

I think chunks are 4MB, but the server might need to allocate them before you can upload the chunk.
https://we.phorge.it/source/arcanist/browse/master/src/upload/ArcanistFileUploader.php has the code to actually use it (It's what being used by the arc upload command).

mturdus
Updated 401 Days Ago

You need to execute the file.allocate first.
If the response contains result->filePHID then you have a "large" file and you need to execute file.querychunks and file.uploadchunk.
If the response does not contain result->filePHID, then you need to execute file.upload.

I have used these examples for unit testing:

file-upload-large.py
import base64
import json
import os
import requests
import sys
from PIL import Image

phabricator_url = sys.argv[1]
api_token = sys.argv[2]
file_path = "/tmp/red.bmp"
image_width = 3500
image_height = 3500

def call_conduit(method, params):
    params['__conduit__'] = {'token': api_token}
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    data = 'params={}&output=json'.format(json.dumps(params))
    response = requests.post(f'{phabricator_url}/api/{method}', data=data, headers=headers)
    print(f'### {method}: {response.text}')
    return json.loads(response.text)

# Generate a red BMP image file
image = Image.new("RGB", (image_width, image_height), "red")
image.save(file_path)

# Read the file data
with open(file_path, "rb") as f:
    file_data = f.read()

# Allocate a new file using file.allocate
allocate_params = {
    "name": "red.bmp",
    "contentLength": len(file_data),
    "contentHash": None,
    "viewPolicy": "public"
}
allocate_result = call_conduit('file.allocate', allocate_params)
file_phid = allocate_result['result']['filePHID']

if file_phid is not None:
    # Query the chunks of the file using file.querychunks
    query_chunks_params = {
        "filePHID": file_phid
    }
    query_chunks_result = call_conduit('file.querychunks', query_chunks_params)

    # Upload the file data using file.uploadchunk
    for chunk in query_chunks_result['result']:
        byte_start = int(chunk['byteStart'])
        byte_end = int(chunk['byteEnd'])
        chunk_data = file_data[byte_start:byte_end]
        chunk_data_base64 = base64.b64encode(chunk_data).decode("utf-8")
        upload_chunk_params = {
            "filePHID": file_phid,
            "byteStart": byte_start,
            "dataEncoding": "base64",
            "data": chunk_data_base64
        }
        upload_chunk_result = call_conduit('file.uploadchunk', upload_chunk_params)

    print(f"File uploaded successfully: {phabricator_url}/file/data/")
else:
    print("This is not a large file.")

# Delete the generated image file
os.remove(file_path)
file-upload-small.py
import base64
import json
import requests
import sys

phabricator_url = sys.argv[1]
api_token = sys.argv[2]
file_path = sys.argv[0];

def call_conduit(method, params):
    params['__conduit__'] = {'token': api_token}
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    data = 'params={}&output=json'.format(json.dumps(params))
    response = requests.post(f'{phabricator_url}/api/{method}', data=data, headers=headers)
    print(response.text)
    return json.loads(response.text)

# Read the file data and encode it as base64
with open(file_path, "rb") as f:
    file_data = f.read()
file_data_base64 = base64.b64encode(file_data).decode("utf-8")

# Allocate a new file using file.allocate
allocate_params = {
    "name": "file-upload.py",
    "contentLength": len(file_data),
    "contentHash": None,
    "viewPolicy": "public"
}
allocate_result = call_conduit('file.allocate', allocate_params)
file_phid = allocate_result['result']['filePHID']

if file_phid is None:
    # Upload the entire file data at once using file.upload
    upload_params = {
        "name": "file-upload.py",
        "data_base64": file_data_base64,
        "viewPolicy": "public"
    }
    upload_result = call_conduit('file.upload', upload_params)

    print(f"File uploaded successfully: {phabricator_url}/file/data/{upload_result['result']}/")
else:
    print("This is not a small file.")

New Answer

Answer

This question has been marked as closed, but you can still leave a new answer.