diff --git a/.travis.yml b/.travis.yml index 10d151c..2fe36e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,8 @@ jobs: os: linux script: - julia --project=tmpsqs -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.add("AWSSQS"); Pkg.test("AWSSQS")' - - stage: "EC2" + - stage: "Online tests" + name: "EC2" julia: 1.0 os: linux before_install: @@ -66,7 +67,7 @@ jobs: @fetch @eval using Pkg rev=ENV["TRAVIS_COMMIT"] @fetch Pkg.add(PackageSpec(url="https://github.com/JuliaCloud/AWSCore.jl", rev=rev)) - @fetch ENV["AWSCORE_EC2"]="true" + @fetch ENV["AWSCORE_INSTANCE_TYPE"]="EC2" @fetch Pkg.build("AWSCore") @fetch @eval using AWSCore @fetch Pkg.test("AWSCore") @@ -79,6 +80,31 @@ jobs: aws ec2 wait instance-terminated --region us-east-1 --instance-ids $INSTANCE_ID fi after_success: skip + - stage: "Online tests" + name: "ECS" + julia: 1.0 + os: linux + before_install: + - pip install --user awscli + script: + - id=$(aws batch submit-job --region us-east-1 --job-name $TRAVIS_COMMIT --job-definition AWSCore-ECS-00001-Job:2 --job-queue AWSCore-ECS-00001-JobQueue --container-overrides "environment=[{name=AWSCORE_BRANCH,value=$TRAVIS_COMMIT}]" --query jobId --output text) + - | + status=$(aws batch describe-jobs --region us-east-1 --jobs $id --query jobs[0].status --output text) + # If the query fails it will either return "None" or status won't be set + while [ "$status" ] && [ "$status" != "None" ] && [ "$status" != "FAILED" ] && [ "$status" != "SUCCEEDED" ]; do + echo "Waiting for job $id, status: $status..." + sleep 30 + status=$(aws batch describe-jobs --region us-east-1 --jobs $id --query jobs[0].status --output text) + done + echo "Job $id finished, status: $status" + - | + # Get test output + logstream=$(aws batch describe-jobs --region us-east-1 --jobs $id --query jobs[0].container.logStreamName --output text) + if [ "$logstream" ]; then + aws logs get-log-events --region us-east-1 --log-group-name /aws/batch/job --log-stream-name $logstream --query events[*].message --output text | tr '\t' '\n' + fi + - test "$status" == "SUCCEEDED" + after_success: skip - stage: "Documentation" julia: 1.0 os: linux diff --git a/test/aws/ecs.yml b/test/aws/ecs.yml new file mode 100644 index 0000000..194126e --- /dev/null +++ b/test/aws/ecs.yml @@ -0,0 +1,198 @@ +# To test, do: +# ``` +# BRANCH="name-of-branch-to-test" +# +# id=$(aws batch submit-job \ +# --job-name $BRANCH \ +# --job-definition AWSCore-ECS-00001-Job:1 \ +# --job-queue AWSCore-ECS-00001-JobQueue \ +# --container-overrides \ +# "environment=[{name=AWSCORE_BRANCH,value=$BRANCH}]" --query jobId --output text) + +# status=$(aws batch describe-jobs --jobs $id --query jobs[0].status --output text) +# while [ "$status" ] && [ "$status" != "FAILED" ] && [ "$status" != "SUCCEEDED" ]; do +# echo "Waiting for job $id, status: $status..." +# sleep 30 +# status=$(aws batch describe-jobs --jobs $id --query jobs[0].status --output text) +# done +# echo "Job $id finished, status: $status" + +# logstream=$(aws batch describe-jobs --jobs $id --query jobs[0].container.logStreamName --output text) +# if [ "$logstream" ]; then +# aws logs get-log-events --log-group-name /aws/batch/job --log-stream-name $logstream --query events[*].message --output text | tr '\t' '\n' +# fi + +# test "$status" == "SUCCEEDED" +# ``` + +AWSTemplateFormatVersion: 2010-09-09 +Description: >- + A bare bones AWS Batch environment for testing with AWSCore + +Parameters: + PublicCIUser: + Description: User which can assume the testing role + Type: String + MaxVCPUs: + Description: >- + The maximum number of VCPUs. Typically this number does not need to be touched + Type: Number + Default: 16 + SubnetId: + Description: >- + The Subnet in the default VPC to launch the ComputeEnvironment into. + Can be found with: + `aws ec2 describe-subnets --filter Name=defaultForAz,Values=true Name=vpc-id,Values=$default_vpc + --query Subnets[0].SubnetId --output text` + Type: AWS::EC2::Subnet::Id + VPCId: + Description: >- + The default VPC in which SubnetId is located. + Can be found with: + `aws ec2 describe-vpcs --filters Name=isDefault,Values=true --query Vpcs[0].VpcId --output text` + Type: AWS::EC2::VPC::Id + +Resources: + ECSTravisPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Users: + - !Sub ${PublicCIUser} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: batch:DescribeJobs + Resource: "*" + - Effect: Allow + Action: logs:GetLogEvents + Resource: + - !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/batch/job* + - Effect: Allow + Action: batch:SubmitJob + Resource: + - !Ref AWSCoreTestJob + - !Ref JobQueue + + AWSCoreTestJob: + Type: AWS::Batch::JobDefinition + Properties: + JobDefinitionName: !Sub ${AWS::StackName}-Job + Type: container + ContainerProperties: + # Job must have a JobRole for `ecs_instance_credentials()` to succeed + JobRoleArn: !Ref JobRole + Memory: 1600 + Command: + - julia + - -e + - using Pkg; Pkg.add(PackageSpec(url=ENV["AWSCORE_URL"], rev=ENV["AWSCORE_BRANCH"])); Pkg.build("AWSCore"); Pkg.test("AWSCore") + Environment: + - Name: AWSCORE_URL + Value: https://github.com/JuliaCloud/AWSCore.jl + - Name: AWSCORE_BRANCH + Value: master + - Name: AWSCORE_INSTANCE_TYPE + Value: ECS + Image: julia:1.0.3 + Vcpus: 1 + Timeout: + AttemptDurationSeconds: 900 + JobRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - ecs-tasks.amazonaws.com # Note: Shouldn't be batch.amazonaws.com + Action: sts:AssumeRole + + ComputeEnvironment: + Type: AWS::Batch::ComputeEnvironment + DependsOn: BatchServiceRole # Removing the ServiceRole before deleting the ComputeEnvironment will cause issues + Properties: + Type: MANAGED + ComputeEnvironmentName: !Ref AWS::StackName + ComputeResources: + Type: SPOT + BidPercentage: 100 + MinvCpus: 0 + MaxvCpus: !Ref MaxVCPUs + InstanceTypes: + - optimal + Subnets: + - !Ref SubnetId + SecurityGroupIds: + - !Ref SecurityGroup + InstanceRole: !Ref IamInstanceProfile + SpotIamFleetRole: !Ref BatchSpotFleetRole + Tags: + Name: !Sub "AWS Batch (${AWS::StackName})" + ServiceRole: !Ref BatchServiceRole + JobQueue: + Type: AWS::Batch::JobQueue + Properties: + JobQueueName: !Sub ${AWS::StackName}-JobQueue + Priority: 1 + ComputeEnvironmentOrder: + - Order: 1 + ComputeEnvironment: !Ref ComputeEnvironment + + IamInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref EcsInstanceRole + EcsInstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2008-10-17 + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role + BatchServiceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: batch.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole + + # http://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html + BatchSpotFleetRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: spotfleet.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole + SecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: EC2 Security Group for instances launched in the VPC by Batch + VpcId: !Ref VPCId + +Outputs: + JobQueueArn: + Value: !Ref JobQueue + JobDefinitionArn: + Value: !Ref AWSCoreTestJob diff --git a/test/runtests.jl b/test/runtests.jl index 7166626..3d41889 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -536,8 +536,14 @@ end @test ex.message == message end -if get(ENV, "AWSCORE_EC2", "false") == "true" +instance_type = get(ENV, "AWSCORE_INSTANCE_TYPE", "") +if instance_type == "EC2" @testset "EC2" begin + @test_nowarn AWSCore.ec2_metadata("instance-id") + @test startswith(AWSCore.ec2_metadata("instance-id"), "i-") + + @test AWSCore.localhost_maybe_ec2() + @test AWSCore.localhost_is_ec2() @test_nowarn AWSCore.ec2_instance_credentials() ec2_creds = AWSCore.ec2_instance_credentials() @test ec2_creds !== nothing @@ -546,7 +552,18 @@ if get(ENV, "AWSCORE_EC2", "false") == "true" @test default_creds.access_key_id == ec2_creds.access_key_id @test default_creds.secret_key == ec2_creds.secret_key end +elseif instance_type == "ECS" + @testset "ECS" begin + @test_nowarn AWSCore.ecs_instance_credentials() + ecs_creds = AWSCore.ecs_instance_credentials() + @test ecs_creds !== nothing + + default_creds = AWSCredentials() + @test default_creds.access_key_id == ecs_creds.access_key_id + @test default_creds.secret_key == ecs_creds.secret_key + end end + end # testset "AWSCore"