From e9c2c5eb55e1012ccce0ce51ac48bed0c0f1d4b7 Mon Sep 17 00:00:00 2001 From: venkata edara Date: Tue, 4 Apr 2017 15:52:53 +0530 Subject: Add support for S3 Multipart Upload API Obj server checks if +segments is there and creates directory to support multi-part upload Updated s3.md to show usage of multi-part upload. Change-Id: I1e8a0dd850f51b2cc5dd2147607e46978dc2f936 Signed-off-by: venkata edara Reviewed-on: https://review.gluster.org/16983 Reviewed-by: Prashanth Pai Tested-by: Prashanth Pai --- doc/markdown/s3.md | 46 +++++++++++++++++++++++++++++++++++++++++++++ gluster/swift/obj/server.py | 11 ++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/doc/markdown/s3.md b/doc/markdown/s3.md index 75a9954..086718e 100644 --- a/doc/markdown/s3.md +++ b/doc/markdown/s3.md @@ -74,6 +74,52 @@ These examples assume you have an account (GlusterFS volume) named `test` and an ```sh ./s3curl.pl --id 'test:tester' --key 'testing' --del -- -k -v -s http://localhost:8080/bucket1 ``` +**Multi-part upload an object to the bucket** + + ***a. Initiate multi part*** +```sh +./s3curl.pl --id 'test:tester' --key 'testing' --post -- "http://localhost:8080/bucket/mobject?uploads" +``` + +***b.upload the multi parts*** +upload all the parts by changing partNumber. get the uploadId from the output of Initiate multipart +```sh +./s3curl.pl --id 'test:tester' --key 'testing' --put xaa -s 'http://localhost:8080/bucket/mobject?uploadId=&partNumber=' +``` + +***c.complete the multi part*** + + ****c.1 List the part numbers and Etags**** +```sh +./s3curl.pl --id 'test:tester' --key 'testing' -- "http://localhost:8080/bucket/mobject?uploadId=" +``` + ****c.2 make a xml file with following format and do a POST request**** + + example: cat ./complete_multi_tag +```xml. + + + 1 + "9dff188990a9831c5255d342895832df" + + + 2 + "72e117e4266579616ad295f399ae10a3" + + + 3 + "85c6e4487367f279926ad5da495a1d20" + + +``` +```sh +./s3curl.pl --id 'test:tester' --key 'testing' --post ./complete_multi_tag -- http://localhost:8080/bucket/mobject?uploadId= +``` + +***d.download the multi part object*** +```sh +./s3curl.pl --id 'test:tester' --key 'testing' -- "http://localhost:8080/bucket/mobject" > output_file +``` **Using boto module in python to access GlusterFS cluster over S3 API** ```python diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py index 3e27cc3..ae225b2 100644 --- a/gluster/swift/obj/server.py +++ b/gluster/swift/obj/server.py @@ -19,7 +19,8 @@ import os from swift.common.swob import HTTPConflict, HTTPNotImplemented from swift.common.utils import public, timing_stats, replication, mkdirs -from swift.common.request_helpers import split_and_validate_path +from swift.common.request_helpers import split_and_validate_path, \ + get_name_and_placement from swift.obj import server from gluster.swift.obj.diskfile import DiskFileManager @@ -135,6 +136,14 @@ class ObjectController(server.ObjectController): @timing_stats() def PUT(self, request): try: + # hack for supporting multi-part. create dir during initialization + content_length = int(request.headers.get('Content-Length', -1)) + authorization = request.headers.get('Authorization', '') + if content_length == 0 and 'AWS' in authorization: + device, partition, account, container, obj, policy = \ + get_name_and_placement(request, 5, 5, True) + if container.endswith("+segments"): + request.headers["Content-Type"] = 'application/directory' # now call swift's PUT method return server.ObjectController.PUT(self, request) except (AlreadyExistsAsFile, AlreadyExistsAsDir): -- cgit