- Cloudflare Developers

Ppolymo1 Click to see attachment

K

Karew•11/3/23, 4:25 AM

Did your token expire?

I

indicozy•11/3/23, 5:51 AM

I'm new on working R2 and S3, it shows that I am Unauthorized. Is it because the cloudflare having issues or am I doing something wrong?

I

indicozy•11/3/23, 7:45 AM

I don't understand why I cannot upload to R2. My code is:

// getS3Client.ts
import { S3Client } from "@aws-sdk/client-s3";
import { getDotenv } from "./getEnv";

let s3Client: S3Client | undefined = undefined;

const env = getDotenv();

export const getS3Client: () => S3Client = () => {
  console.log(env);
  if (s3Client === undefined) {
    s3Client = new S3Client({
      region: "auto",
      endpoint: `https://${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
      credentials: {
        accessKeyId: env.R2_ACCESS_KEY_ID,
        secretAccessKey: env.R2_SECRET_ACCESS_KEY,
      },
    });
    return s3Client;
  }
  return s3Client;
};

// getS3Client.ts
import { S3Client } from "@aws-sdk/client-s3";
import { getDotenv } from "./getEnv";

let s3Client: S3Client | undefined = undefined;

const env = getDotenv();

export const getS3Client: () => S3Client = () => {
  console.log(env);
  if (s3Client === undefined) {
    s3Client = new S3Client({
      region: "auto",
      endpoint: `https://${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
      credentials: {
        accessKeyId: env.R2_ACCESS_KEY_ID,
        secretAccessKey: env.R2_SECRET_ACCESS_KEY,
      },
    });
    return s3Client;
  }
  return s3Client;
};

// getS3Client.ts
import { S3Client } from "@aws-sdk/client-s3";
import { getDotenv } from "./getEnv";

let s3Client: S3Client | undefined = undefined;

const env = getDotenv();

export const getS3Client: () => S3Client = () => {
  console.log(env);
  if (s3Client === undefined) {
    s3Client = new S3Client({
      region: "auto",
      endpoint: `https://${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
      credentials: {
        accessKeyId: env.R2_ACCESS_KEY_ID,
        secretAccessKey: env.R2_SECRET_ACCESS_KEY,
      },
    });
    return s3Client;
  }
  return s3Client;
};

// getS3Client.ts
import { S3Client } from "@aws-sdk/client-s3";
import { getDotenv } from "./getEnv";

let s3Client: S3Client | undefined = undefined;

const env = getDotenv();

export const getS3Client: () => S3Client = () => {
  console.log(env);
  if (s3Client === undefined) {
    s3Client = new S3Client({
      region: "auto",
      endpoint: `https://${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
      credentials: {
        accessKeyId: env.R2_ACCESS_KEY_ID,
        secretAccessKey: env.R2_SECRET_ACCESS_KEY,
      },
    });
    return s3Client;
  }
  return s3Client;
};

const s3Client = getS3Client();
const env = getDotenv();

const resizeImage = async (str: string) => {
  // TODO:
  await sharp(str).webp({ quality: 75 }).toBuffer();
};

export const uploadDirectlyOne = async (path: string, body: string) => {
  // TODO: test it
  console.log("uploading:", path);
  const buf = Buffer.from(body, "utf8");

  const command = new PutObjectCommand({
    Bucket: env.S3_BUCKET,
    Key: path,
    Body: buf,
  });
  try {
    const response = await s3Client.send(command);
    console.log("uploaded:", path);
  } catch (err) {
    console.error(err);
    throw err;
  }
};

const s3Client = getS3Client();
const env = getDotenv();

const resizeImage = async (str: string) => {
  // TODO:
  await sharp(str).webp({ quality: 75 }).toBuffer();
};

export const uploadDirectlyOne = async (path: string, body: string) => {
  // TODO: test it
  console.log("uploading:", path);
  const buf = Buffer.from(body, "utf8");

  const command = new PutObjectCommand({
    Bucket: env.S3_BUCKET,
    Key: path,
    Body: buf,
  });
  try {
    const response = await s3Client.send(command);
    console.log("uploaded:", path);
  } catch (err) {
    console.error(err);
    throw err;
  }
};

const s3Client = getS3Client();
const env = getDotenv();

const resizeImage = async (str: string) => {
  // TODO:
  await sharp(str).webp({ quality: 75 }).toBuffer();
};

export const uploadDirectlyOne = async (path: string, body: string) => {
  // TODO: test it
  console.log("uploading:", path);
  const buf = Buffer.from(body, "utf8");

  const command = new PutObjectCommand({
    Bucket: env.S3_BUCKET,
    Key: path,
    Body: buf,
  });
  try {
    const response = await s3Client.send(command);
    console.log("uploaded:", path);
  } catch (err) {
    console.error(err);
    throw err;
  }
};

const s3Client = getS3Client();
const env = getDotenv();

const resizeImage = async (str: string) => {
  // TODO:
  await sharp(str).webp({ quality: 75 }).toBuffer();
};

export const uploadDirectlyOne = async (path: string, body: string) => {
  // TODO: test it
  console.log("uploading:", path);
  const buf = Buffer.from(body, "utf8");

  const command = new PutObjectCommand({
    Bucket: env.S3_BUCKET,
    Key: path,
    Body: buf,
  });
  try {
    const response = await s3Client.send(command);
    console.log("uploaded:", path);
  } catch (err) {
    console.error(err);
    throw err;
  }
};

V

vvo•11/3/23, 11:30 AM

Hey there, we're unable to assign custom domains to R2 buckets. Where is this situation this reflected on your status pages? Thanks.

K

Kirmola•11/3/23, 12:52 PM

Hi. for some reason, my credentials are being reported as unauthroised by boto.

def saveAndPushToR2(self, filename):
        # Use self.engine
        currentDateFileFormat = datetime.now().strftime("%d_%m_%Y")
        # filename = f"{category}_{currentDateFileFormat}.json"
        s3 = boto3.resource('s3',
                            endpoint_url=f'https://{environ.get("ACCOUNT_ID")}.r2.cloudflarestorage.com',
                            aws_access_key_id=environ.get("ACCESS_KEY_ID"),
                            aws_secret_access_key=environ.get(
                                "SECRET_ACCESS_KEY")
                            )
        with open(filename, "r"):
            # s3.Bucket(environ["BUCKET_NAME"]).put_object(
            #     Key=filename, Body=f, ContentType="application/json"
            # )
            s3.Bucket(environ["BUCKET_NAME"]).upload_file(
                filename, filename, ExtraArgs={'ContentType': "application/json"}
            )
            logging.info(
                f"{filename} uploaded successfully to Cloudflare R2.").

def saveAndPushToR2(self, filename):
        # Use self.engine
        currentDateFileFormat = datetime.now().strftime("%d_%m_%Y")
        # filename = f"{category}_{currentDateFileFormat}.json"
        s3 = boto3.resource('s3',
                            endpoint_url=f'https://{environ.get("ACCOUNT_ID")}.r2.cloudflarestorage.com',
                            aws_access_key_id=environ.get("ACCESS_KEY_ID"),
                            aws_secret_access_key=environ.get(
                                "SECRET_ACCESS_KEY")
                            )
        with open(filename, "r"):
            # s3.Bucket(environ["BUCKET_NAME"]).put_object(
            #     Key=filename, Body=f, ContentType="application/json"
            # )
            s3.Bucket(environ["BUCKET_NAME"]).upload_file(
                filename, filename, ExtraArgs={'ContentType': "application/json"}
            )
            logging.info(
                f"{filename} uploaded successfully to Cloudflare R2.").

def saveAndPushToR2(self, filename):
        # Use self.engine
        currentDateFileFormat = datetime.now().strftime("%d_%m_%Y")
        # filename = f"{category}_{currentDateFileFormat}.json"
        s3 = boto3.resource('s3',
                            endpoint_url=f'https://{environ.get("ACCOUNT_ID")}.r2.cloudflarestorage.com',
                            aws_access_key_id=environ.get("ACCESS_KEY_ID"),
                            aws_secret_access_key=environ.get(
                                "SECRET_ACCESS_KEY")
                            )
        with open(filename, "r"):
            # s3.Bucket(environ["BUCKET_NAME"]).put_object(
            #     Key=filename, Body=f, ContentType="application/json"
            # )
            s3.Bucket(environ["BUCKET_NAME"]).upload_file(
                filename, filename, ExtraArgs={'ContentType': "application/json"}
            )
            logging.info(
                f"{filename} uploaded successfully to Cloudflare R2.").

def saveAndPushToR2(self, filename):
        # Use self.engine
        currentDateFileFormat = datetime.now().strftime("%d_%m_%Y")
        # filename = f"{category}_{currentDateFileFormat}.json"
        s3 = boto3.resource('s3',
                            endpoint_url=f'https://{environ.get("ACCOUNT_ID")}.r2.cloudflarestorage.com',
                            aws_access_key_id=environ.get("ACCESS_KEY_ID"),
                            aws_secret_access_key=environ.get(
                                "SECRET_ACCESS_KEY")
                            )
        with open(filename, "r"):
            # s3.Bucket(environ["BUCKET_NAME"]).put_object(
            #     Key=filename, Body=f, ContentType="application/json"
            # )
            s3.Bucket(environ["BUCKET_NAME"]).upload_file(
                filename, filename, ExtraArgs={'ContentType': "application/json"}
            )
            logging.info(
                f"{filename} uploaded successfully to Cloudflare R2.").

K

Kirmola•11/3/23, 12:53 PM

 raise S3UploadFailedError(
boto3.exceptions.S3UploadFailedError: Failed to upload prepositions_03_11_2023.json to production/prepositions_03_11_2023.json: An error occurred (Unauthorized) when calling the PutObject operation: Unauthorized
2023-11-03 18:17:55 [scrapy.extensions.feedexport] INFO: Stored json feed (49 items) in: output.json

 raise S3UploadFailedError(
boto3.exceptions.S3UploadFailedError: Failed to upload prepositions_03_11_2023.json to production/prepositions_03_11_2023.json: An error occurred (Unauthorized) when calling the PutObject operation: Unauthorized
2023-11-03 18:17:55 [scrapy.extensions.feedexport] INFO: Stored json feed (49 items) in: output.json

 raise S3UploadFailedError(
boto3.exceptions.S3UploadFailedError: Failed to upload prepositions_03_11_2023.json to production/prepositions_03_11_2023.json: An error occurred (Unauthorized) when calling the PutObject operation: Unauthorized
2023-11-03 18:17:55 [scrapy.extensions.feedexport] INFO: Stored json feed (49 items) in: output.json

 raise S3UploadFailedError(
boto3.exceptions.S3UploadFailedError: Failed to upload prepositions_03_11_2023.json to production/prepositions_03_11_2023.json: An error occurred (Unauthorized) when calling the PutObject operation: Unauthorized
2023-11-03 18:17:55 [scrapy.extensions.feedexport] INFO: Stored json feed (49 items) in: output.json

this is error

K

Kirmola•11/3/23, 12:54 PM

my code was working perfectly until today, not sure what happened

E

EchoSevenTz•11/3/23, 4:02 PM

Hi,
if I want to access R2 bucket that has custom domain registered for example myr2.com and want to access it from mydomain.com. I set https://mydomain.com in allowed origins in CORS control panel, but still get " Cross-Origin Request Blocked: The Same Origin Policy disallows reading the remote resource at https://my2r.com/Barutana_Model_700k.glb. (Reason: CORS header ‘Access-Control-Allow-Origin’ missing). Status code: 200.

Web Hosting, Domain Name Registration - MyDomain.com

Small business web hosting offering additional business services such as: domain name registrations, email accounts, web services, and various small business solutions.

KKarew Did your token expire?

P

polymo1OP•11/3/23, 4:29 PM

i've rolled the tokens multiple times and I set my tokens to not expire

EEchoSevenTz Hi, if I want to access R2 bucket that has custom domain registered for example ...

K

kev-ac•11/3/23, 7:32 PM

Please verify that the Cors headers are actually set in the response. Its possible they haven't been applied yet because of the degraded state of CFs control plane.

Kkev-ac Please verify that the Cors headers are actually set in the response. Its possib...

E

EchoSevenTz•11/3/23, 7:37 PM

In respons there are no headers related to cross origin. On page that requested data from R2 I included header “ origin: https://mydomain.com ” as I understod that cors will not respond with cors headers if there is no correct header in request

Web Hosting, Domain Name Registration - MyDomain.com

Small business web hosting offering additional business services such as: domain name registrations, email accounts, web services, and various small business solutions.

EEchoSevenTz In respons there are no headers related to cross origin. On page that requested ...

K

kev-ac•11/3/23, 7:55 PM

Browsers add the Origin header automatically on cross-origin requests.

O

orkrist•11/3/23, 7:56 PM

hello. is there a problem with authentication? I have not changed anything, and I am getting unauthorized from both go-sdk and aws cli.

Oorkrist hello. is there a problem with authentication? I have not changed anything, and ...

K

Karew•11/3/23, 8:50 PM

Did your token expire?

O

orkrist•11/3/23, 8:53 PM

nope

O

orkrist•11/3/23, 8:53 PM

I recreated it

O

orkrist•11/3/23, 8:53 PM

I mean I created a new one

O

orkrist•11/3/23, 8:53 PM

no luck

O

orkrist•11/3/23, 8:54 PM

this is serious, the rest of my team also reports problems

O

orkrist•11/3/23, 8:54 PM

am I the only one?

O

orkrist•11/3/23, 8:54 PM

(and thanks for responding @Karew )

O

orkrist•11/3/23, 9:09 PM

how does the support for r2 work? how can I get help? I mean technically I am paying for data which I cant access in any way.

O

orkrist•11/3/23, 9:10 PM

only a bit desperate

K

kev-ac•11/3/23, 10:34 PM

Is there a way similar to S3 to limit the upload size for a multipart upload client-side in R2? In AWS there seems to be a way to limit the size with an IAM role, which R2 doesn't have. Maybe I'm missing an option..

My current solution is to create the multipart upload and then generating the presigned URLs something like this:

// Size is sent over by the client. It is checked against the maximum allowed file size on the server
const chunkCount = Math.ceil(size / FILE_CHUNK_SIZE_BYTES);
const fileKey = "my/r2/file_key";
const uploadSessionId = "R2_MULTIPART_SESSION_ID";
const presignedUrls = [];

for(let x = 1; x <= chunkCount; x++) {
  presignedUrls.push(await r2.getPresignedChunkUrl(fileKey, uploadSessionId, x, (60 * 60 * 24), x < chunkCount ? FILE_CHUNK_SIZE_BYTES : (size % FILE_CHUNK_SIZE_BYTES)));
}

// Size is sent over by the client. It is checked against the maximum allowed file size on the server
const chunkCount = Math.ceil(size / FILE_CHUNK_SIZE_BYTES);
const fileKey = "my/r2/file_key";
const uploadSessionId = "R2_MULTIPART_SESSION_ID";
const presignedUrls = [];

for(let x = 1; x <= chunkCount; x++) {
  presignedUrls.push(await r2.getPresignedChunkUrl(fileKey, uploadSessionId, x, (60 * 60 * 24), x < chunkCount ? FILE_CHUNK_SIZE_BYTES : (size % FILE_CHUNK_SIZE_BYTES)));
}

// Size is sent over by the client. It is checked against the maximum allowed file size on the server
const chunkCount = Math.ceil(size / FILE_CHUNK_SIZE_BYTES);
const fileKey = "my/r2/file_key";
const uploadSessionId = "R2_MULTIPART_SESSION_ID";
const presignedUrls = [];

for(let x = 1; x <= chunkCount; x++) {
  presignedUrls.push(await r2.getPresignedChunkUrl(fileKey, uploadSessionId, x, (60 * 60 * 24), x < chunkCount ? FILE_CHUNK_SIZE_BYTES : (size % FILE_CHUNK_SIZE_BYTES)));
}

// Size is sent over by the client. It is checked against the maximum allowed file size on the server
const chunkCount = Math.ceil(size / FILE_CHUNK_SIZE_BYTES);
const fileKey = "my/r2/file_key";
const uploadSessionId = "R2_MULTIPART_SESSION_ID";
const presignedUrls = [];

for(let x = 1; x <= chunkCount; x++) {
  presignedUrls.push(await r2.getPresignedChunkUrl(fileKey, uploadSessionId, x, (60 * 60 * 24), x < chunkCount ? FILE_CHUNK_SIZE_BYTES : (size % FILE_CHUNK_SIZE_BYTES)));
}

async getPresignedChunkUrl(key: string, uploadSessionId: string, partNumber: number, expiresInSec: number, contentLength: number) {
    try {
      const command = new UploadPartCommand({
        Bucket: this.bucket,
        Key: key,
        UploadId: uploadSessionId,
        PartNumber: partNumber,
        ContentLength: contentLength
      });

      return await getSignedUrl(this.client, command, {expiresIn: expiresInSec});
    }
    catch (e: any) {
      console.error(e?.message);
    }
    return null;
  }

async getPresignedChunkUrl(key: string, uploadSessionId: string, partNumber: number, expiresInSec: number, contentLength: number) {
    try {
      const command = new UploadPartCommand({
        Bucket: this.bucket,
        Key: key,
        UploadId: uploadSessionId,
        PartNumber: partNumber,
        ContentLength: contentLength
      });

      return await getSignedUrl(this.client, command, {expiresIn: expiresInSec});
    }
    catch (e: any) {
      console.error(e?.message);
    }
    return null;
  }

async getPresignedChunkUrl(key: string, uploadSessionId: string, partNumber: number, expiresInSec: number, contentLength: number) {
    try {
      const command = new UploadPartCommand({
        Bucket: this.bucket,
        Key: key,
        UploadId: uploadSessionId,
        PartNumber: partNumber,
        ContentLength: contentLength
      });

      return await getSignedUrl(this.client, command, {expiresIn: expiresInSec});
    }
    catch (e: any) {
      console.error(e?.message);
    }
    return null;
  }

async getPresignedChunkUrl(key: string, uploadSessionId: string, partNumber: number, expiresInSec: number, contentLength: number) {
    try {
      const command = new UploadPartCommand({
        Bucket: this.bucket,
        Key: key,
        UploadId: uploadSessionId,
        PartNumber: partNumber,
        ContentLength: contentLength
      });

      return await getSignedUrl(this.client, command, {expiresIn: expiresInSec});
    }
    catch (e: any) {
      console.error(e?.message);
    }
    return null;
  }

Every chunk has the FILE_CHUNK_SIZE_BYTES as the ContentLength except the last one which is the remainder of bytes.

It works but it feels clumsy.

E

Evil Morty•11/3/23, 10:45 PM

Hello what is the problem here?

Pprender most of requests are 70-180ms which is also a bit slow for me, but sometimes it ...

K

Karew•11/4/23, 2:07 AM

Are the files really new or very cold? That is not unusual for the very first file reads

K

Karew•11/4/23, 2:07 AM

But also there is currently a company-wide outage incident, so maybe don't test extensively right now

K

Karew•11/4/23, 2:12 AM

Unfortuantely, latency is hard to diagnose. It can depend on a lot of factors at the time you make the request. Are you using a custom domain and the CDN? What kind of request are you making? Signed?

L

lottor•11/4/23, 2:45 AM

hi, r2 is down? im getting error 401 since 2 hours ago ... i cant view/upload files

M

matcha•11/4/23, 2:46 AM

Hey guys, are you having unauthorized too on cloudflare R2?

Mmatcha Hey guys, are you having unauthorized too on cloudflare R2?

L

lottor•11/4/23, 2:46 AM

my site has that error .. since 2 hours ago ..

U

Unsmart•11/4/23, 2:47 AM

R2 is very much still up for me are you using workers bindings or custom domains or s3 api?

M

matcha•11/4/23, 2:47 AM

I use s3 api, it's always return 401 unauthorized when submitting HEAD request

L

lottor•11/4/23, 2:48 AM

im using s3 api too

U

Unsmart•11/4/23, 2:48 AM

Hmm s3 api works fine for me. If you go to the cloudflare dash do you see the key you are using? Its possible the key you are using has expired

M

matcha•11/4/23, 2:49 AM

I had this issue raise to me since yesterday, the key has been rolled out multiple times but still having same response from r2

U

Unsmart•11/4/23, 2:51 AM

Due to the outage of cloudflares core datacenter its possible updates to api keys may have been delayed things are looking a lot better now so I would try generating a fresh key and seeing if that works. Otherwise you might have to wait until the incident is fully resolved as it could still be having issues syncing api key data to edge datacenters

M

matcha•11/4/23, 2:54 AM

Ah i see

L

lottor•11/4/23, 3:08 AM

i created a new key but it doesn't work.. I guess I'll have to wait..

G

grizzlebizzle•11/4/23, 4:23 AM

just started looking at using R2 for a project a week ago - is this kind of outage common? Looks like it started a couple days ago now and is still ongoing? I was able to sign up and create a bucket but getting all kinds of errors in the dashboard (internal error 10001) after doing that so I guess maybe I need to wait till the outage is resolved to get any further. I got a URL for the bucket but no key info or anything else - course I could be missing it since I haven't used it before. I'm assuming I can get that info in the area under where the URL is specified - which is blank for me.

F

fernando•11/4/23, 4:48 AM

Hi, I am trying to upload a file to a bucket using a presigned PutObjectCommand url, but I get this error

<Error>
    <Code>InternalError</Code>
    <Message>We encountered an internal error. Please try again.</Message>
</Error>

<Error>
    <Code>InternalError</Code>
    <Message>We encountered an internal error. Please try again.</Message>
</Error>

<Error>
    <Code>InternalError</Code>
    <Message>We encountered an internal error. Please try again.</Message>
</Error>

<Error>
    <Code>InternalError</Code>
    <Message>We encountered an internal error. Please try again.</Message>
</Error>

I am getting the URL like this:

getSignedUrl(s3, new PutObjectCommand({
        Bucket: bucket,
        Key: `${fileName}`
    }), {    expiresIn: 600 })

getSignedUrl(s3, new PutObjectCommand({
        Bucket: bucket,
        Key: `${fileName}`
    }), {    expiresIn: 600 })

getSignedUrl(s3, new PutObjectCommand({
        Bucket: bucket,
        Key: `${fileName}`
    }), {    expiresIn: 600 })

getSignedUrl(s3, new PutObjectCommand({
        Bucket: bucket,
        Key: `${fileName}`
    }), {    expiresIn: 600 })

and making an HTTP PUT Request on Postman. Is there anywhere where I can get more information about what I am doing wrong?

K

Kirmola•11/4/23, 5:00 AM

Looks like outage affected R2 too. All secret keys are returning unauthorised here I see

Similar Threads