From 59854f364c3ffca1f57954dcbc0e90fd868504f4 Mon Sep 17 00:00:00 2001 From: bethac07 Date: Fri, 11 Sep 2026 08:49:00 +0200 Subject: [PATCH] move ls to run-worker, one level only --- worker/cp-worker.py | 7 ------- worker/run-worker.sh | 4 +++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/worker/cp-worker.py b/worker/cp-worker.py index e54b6b8..1e9841d 100644 --- a/worker/cp-worker.py +++ b/worker/cp-worker.py @@ -111,13 +111,6 @@ def printandlog(text,logger): def runCellProfiler(message): s3client=boto3.client('s3') - #List the directories in the bucket- this prevents a strange s3fs error - rootlist=os.listdir(DATA_ROOT) - for eachSubDir in rootlist: - subDirName=os.path.join(DATA_ROOT,eachSubDir) - if os.path.isdir(subDirName): - trashvar=os.system(f'ls {subDirName}') - # Configure the logs logger = logging.getLogger(__name__) diff --git a/worker/run-worker.sh b/worker/run-worker.sh index 02471c3..85441f2 100644 --- a/worker/run-worker.sh +++ b/worker/run-worker.sh @@ -22,17 +22,19 @@ aws ec2 create-tags --resources $VOL_0_ID --tags Key=Name,Value=${APP_NAME}Worke VOL_1_ID=$(aws ec2 describe-instance-attribute --instance-id $MY_INSTANCE_ID --attribute blockDeviceMapping --output text --query BlockDeviceMappings[1].Ebs.[VolumeId]) aws ec2 create-tags --resources $VOL_1_ID --tags Key=Name,Value=${APP_NAME}Worker -# 2. MOUNT S3 +# 2. MOUNT S3 (and ls for s3fs bug - issues/214) if [[ ${DOWNLOAD_FILES} == 'False' ]]; then echo "Mounting S3 using S3FS" if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then echo "Using role credentials to mount S3" s3fs $SOURCE_BUCKET /home/ubuntu/bucket -o iam_role + ls /home/ubuntu/bucket/ else echo "Using user credentials to mount S3" echo $AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY > /credentials.txt chmod 600 /credentials.txt s3fs $SOURCE_BUCKET /home/ubuntu/bucket -o passwd_file=/credentials.txt + ls /home/ubuntu/bucket/ fi fi