Good day,
Info:
rancher-server version: v1.6.2
rancher-ebs version: v0.8.1
I have test stack:
version: '2'
volumes:
data:
driver: rancher-ebs
driver_opts:
size: 10
volumeType: gp2
snapshotID: snap-xxx
ec2_az: xxx
services:
mysql:
container_name: mysql
image: xxx
volumes:
- data:/var/lib/mysql/
command: sleep infinity
After removing stack the volume still exists in AWS in state ‘Available’.
The aws account policy is:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "xxx",
"Effect": "Allow",
"Action": [
"ec2:AttachVolume",
"ec2:CreateVolume",
"ec2:DeleteVolume",
"ec2:DescribeSnapshots",
"ec2:DescribeTags",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"ec2:CreateTags",
"ec2:DeleteTags",
"ec2:CreateSnapshot",
"ec2:DeleteSnapshot",
"ec2:CreateTags"
],
"Resource": [
"*"
]
}
]
}
I compared ebs logs between working and non-working ebs (both have same versions).
Working ebs log:
...
time="2017-06-29T09:11:47Z" level=info msg=create.request name="web-db-10g-3_data_8eb99" options=map[ec2_az:us-east-1e size:10 snapshotID:snap-xxx volumeType:gp2]
time="2017-06-29T09:12:14Z" level=info msg="Updating volume web-db-10g-3_data_8eb99 (1v16650) rancher-ebs:1sd2 on 1h1, state inactive => updating-inactive" err=<nil>
time="2017-06-29T09:12:14Z" level=info msg=create.response
time="2017-06-29T09:12:14Z" level=info msg=attach.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:19Z" level=info msg=attach.response
time="2017-06-29T09:12:19Z" level=info msg=mount.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:20Z" level=info msg=mount.response mountpoint="/var/lib/rancher/volumes/rancher-ebs/web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:20Z" level=info msg=unmount.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:20Z" level=info msg=unmount.response
time="2017-06-29T09:12:20Z" level=info msg=mount.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:20Z" level=info msg="web-db-10g-3_data_8eb99 already mounted on /var/lib/rancher/volumes/rancher-ebs/web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:20Z" level=info msg=mount.response mountpoint="/var/lib/rancher/volumes/rancher-ebs/web-db-10g-3_data_8eb99"
time="2017-06-29T09:12:21Z" level=error msg="Mounted but not registered in Docker: /var/lib/rancher/volumes/rancher-ebs/web-db-10g-3_data_8eb99-staging"
time="2017-06-29T09:12:21Z" level=info msg="Unmounting /var/lib/rancher/volumes/rancher-ebs/web-db-10g-3_data_8eb99-staging"
time="2017-06-29T09:13:04Z" level=info msg="container 51a7ddfea950d03171c82de05a9e8d1b3bbfa578f6cad85e8e6059d713e3bc40 destroyed"
time="2017-06-29T09:13:05Z" level=info msg=unmount.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:13:05Z" level=info msg=unmount.response
time="2017-06-29T09:13:05Z" level=info msg="container bccbbc4d3dc7ad2586caa08302368b4519cb52b5fa502d032b4f9c555cc6ee3a destroyed"
time="2017-06-29T09:13:05Z" level=info msg="container 65221b075399c228861343195bbc2d6cc1ff3f753863628d88eb50db0f832fdc destroyed"
time="2017-06-29T09:13:06Z" level=info msg="Unmounting /var/lib/rancher/volumes/rancher-ebs/web-db-10g-3_data_8eb99"
time="2017-06-29T09:13:06Z" level=info msg=remove.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:13:07Z" level=error msg=remove.response error="Failed to delete volume vol-xxx, current state: in-use is not available. Should retry"
time="2017-06-29T09:13:07Z" level=info msg=remove.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:13:07Z" level=error msg=remove.response error="Failed to delete volume vol-xxx, current state: in-use is not available. Should retry"
time="2017-06-29T09:13:17Z" level=info msg=remove.request name="web-db-10g-3_data_8eb99"
time="2017-06-29T09:13:18Z" level=info msg=remove.response
Non-working ebs log:
...
time="2017-06-29T10:45:02Z" level=info msg=create.request name="ebs-test-5_data_fe012" options=map[ec2_az:us-east-1a size:10 snapshotID:snap-xxx volumeType:gp2]
time="2017-06-29T10:45:23Z" level=info msg="Updating volume ebs-test-5_data_fe012 (1v425674) rancher-ebs:1sd21 on 1h128, state inactive => updating-inactive" err=<nil>
time="2017-06-29T10:45:23Z" level=info msg=create.response
time="2017-06-29T10:45:23Z" level=info msg=attach.request name="ebs-test-5_data_fe012"
time="2017-06-29T10:45:28Z" level=info msg=attach.response
time="2017-06-29T10:45:28Z" level=info msg=mount.request name="ebs-test-5_data_fe012"
time="2017-06-29T10:45:29Z" level=info msg=mount.response mountpoint="/var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012"
time="2017-06-29T10:45:29Z" level=info msg=unmount.request name="ebs-test-5_data_fe012"
time="2017-06-29T10:45:29Z" level=info msg=unmount.response
time="2017-06-29T10:45:29Z" level=info msg=mount.request name="ebs-test-5_data_fe012"
time="2017-06-29T10:45:29Z" level=info msg="ebs-test-5_data_fe012 already mounted on /var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012"
time="2017-06-29T10:45:29Z" level=info msg=mount.response mountpoint="/var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012"
time="2017-06-29T10:45:30Z" level=error msg="Mounted but not registered in Docker: /var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012-staging"
time="2017-06-29T10:45:30Z" level=info msg="Unmounting /var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012-staging"
time="2017-06-29T10:48:51Z" level=info msg=unmount.request name="ebs-test-5_data_fe012"
time="2017-06-29T10:48:51Z" level=info msg=unmount.response
time="2017-06-29T10:48:52Z" level=info msg="container 116725fa8064646e82cf388c35536b4586df853e0ef978224daff4e6ac347643 destroyed"
time="2017-06-29T10:48:52Z" level=error msg="Mounted but not registered in Docker: /var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012"
time="2017-06-29T10:48:52Z" level=info msg="Unmounting /var/lib/rancher/volumes/rancher-ebs/ebs-test-5_data_fe012"
As you see, the part about removing vol is missed in non-working ebs log.
Still can’t found solution, what do you think, how it can be resolved?
p.s. have much environments in rancher-server now, so recreate it from scratch is not good idea.