[ Aug 16 04:35:01 Enabled. ] [ Aug 16 04:35:01 Rereading configuration. ] [ Aug 16 04:35:02 Rereading configuration. ] [ Aug 16 04:35:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:04.515262115Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:04.516403153Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:04.516415623Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:04.537023072Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:04.673705327Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:58.984975227Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39884"} {"msg":"region d1a8f3b4-372d-4a91-90b3-fb051a2f348b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.010231358Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.015815113Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.020369393Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions","method":"POST","req_id":"71762eeb-ee8b-4c48-b6fc-6c760d8dc065","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16084,"response_code":"200"} {"msg":"zfs set reservation of 1342177280 for oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.028164341Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.028212353Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"d1a8f3b4-372d-4a91-90b3-fb051a2f348b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.105353058Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"d1a8f3b4-372d-4a91-90b3-fb051a2f348b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d1a8f3b4-372d-4a91-90b3-fb051a2f348b\" \"--data\" \"/data/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.107432078Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"d1a8f3b4-372d-4a91-90b3-fb051a2f348b","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.279021827Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"d1a8f3b4-372d-4a91-90b3-fb051a2f348b","component":"worker"} {"msg":"region d1a8f3b4-372d-4a91-90b3-fb051a2f348b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.27911834Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.279289464Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.279709484Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.285599166Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.301267265Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.304915233Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.31303954Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.315082619Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.315153761Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.315204882Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.320561952Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.325136782Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:35:59.394449409Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions","method":"POST","req_id":"0cb92d48-fab8-4aeb-a961-c2257dfecddc","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.418772267Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","method":"GET","req_id":"6a8d64b4-a6a1-42cc-b66b-aefc42f712be","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"path is \"/data/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.421266737Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"dataset is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.430168163Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"region d1a8f3b4-372d-4a91-90b3-fb051a2f348b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.438870903Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.439032637Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.439110669Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","method":"DELETE","req_id":"456cb327-6cef-4046-99b9-5c20c72de1a3","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18322,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-d1a8f3b4-372d-4a91-90b3-fb051a2f348b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.440126043Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.441558098Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","method":"GET","req_id":"7fbcdfa1-3393-4e97-97bc-2103be0be013","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":28,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.444191082Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.453341853Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"d1a8f3b4-372d-4a91-90b3-fb051a2f348b","component":"worker"} {"msg":"region d1a8f3b4-372d-4a91-90b3-fb051a2f348b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.500901213Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:00.587709182Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/d1a8f3b4-372d-4a91-90b3-fb051a2f348b","method":"GET","req_id":"718d6006-8786-45fe-bd7c-b7bf9c5bb455","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":"200"} {"msg":"region 36fcdb41-0c7a-4daf-8916-606855849fb8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.846062771Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.846197104Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions","method":"POST","req_id":"e2e71b67-42e5-4c1e-b554-8fafc3ead81b","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":206,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.846220554Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.855066328Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.855108079Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"36fcdb41-0c7a-4daf-8916-606855849fb8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.904563995Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"36fcdb41-0c7a-4daf-8916-606855849fb8","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"36fcdb41-0c7a-4daf-8916-606855849fb8\" \"--data\" \"/data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.904617586Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"36fcdb41-0c7a-4daf-8916-606855849fb8","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.969735771Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"region":"36fcdb41-0c7a-4daf-8916-606855849fb8","component":"worker"} {"msg":"region 36fcdb41-0c7a-4daf-8916-606855849fb8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.969788052Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.969880334Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.971320169Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.975564882Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.982193192Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.98498771Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.985325928Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.985378349Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.98542157Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.985461131Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.988675489Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:01.993509546Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:36:02.188414269Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions","method":"POST","req_id":"c8ba58e9-2ac6-4006-90da-1b013f16f5f5","remote_addr":"[fd00:1122:3344:101::a]:39884","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.362735616Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34761"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.36289951Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","method":"GET","req_id":"865d7758-3f06-4361-80af-238293aef56c","remote_addr":"[fd00:1122:3344:101::a]:34761","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":49,"response_code":"200"} {"msg":"path is \"/data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.368144467Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"dataset is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.377296118Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"snapshot is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8@e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.386751147Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\", \"e59028bb-7f16-4e41-80d8-a197ded34d5e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.386781547Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.400716244Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/36fcdb41-0c7a-4daf-8916-606855849fb8/snapshots/e59028bb-7f16-4e41-80d8-a197ded34d5e","method":"GET","req_id":"186423a4-4f32-48db-a71b-8c0e392d89a6","remote_addr":"[fd00:1122:3344:101::a]:34761","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":35239,"response_code":"200"} {"msg":"path is \"/data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.403826889Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"dataset is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.41292656Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"snapshot is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8@e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.421311092Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\", \"e59028bb-7f16-4e41-80d8-a197ded34d5e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.421348653Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 36fcdb41-0c7a-4daf-8916-606855849fb8-e59028bb-7f16-4e41-80d8-a197ded34d5e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.430784281Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"applying SMF actions for region 36fcdb41-0c7a-4daf-8916-606855849fb8 running snapshot e59028bb-7f16-4e41-80d8-a197ded34d5e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.431394816Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.431424847Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/36fcdb41-0c7a-4daf-8916-606855849fb8/snapshots/e59028bb-7f16-4e41-80d8-a197ded34d5e/run","method":"POST","req_id":"90cec4c3-d11a-42eb-ab06-a136ebbda06a","remote_addr":"[fd00:1122:3344:101::a]:34761","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27645,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-36fcdb41-0c7a-4daf-8916-606855849fb8-e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.438525308Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-36fcdb41-0c7a-4daf-8916-606855849fb8-e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.449189057Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-36fcdb41-0c7a-4daf-8916-606855849fb8-e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.464408434Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8/.zfs/snapshot/e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.464776013Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.464840665Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.464871406Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.464914757Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.464960398Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.471913086Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.477176703Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"worker"} {"msg":"region 36fcdb41-0c7a-4daf-8916-606855849fb8 running snapshot e59028bb-7f16-4e41-80d8-a197ded34d5e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.477196434Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"path is \"/data/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.575286956Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"dataset is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.58373929Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"datafile"} {"msg":"snapshot is oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8@e59028bb-7f16-4e41-80d8-a197ded34d5e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.592391449Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_70d53ccc-0896-4d3c-9c34-3a4ffe4d8ee6/crucible/regions/36fcdb41-0c7a-4daf-8916-606855849fb8\", \"e59028bb-7f16-4e41-80d8-a197ded34d5e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.59243504Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T04:41:20.600588938Z","hostname":"oxz_crucible_ec2e64cb-ae4b-4d26-8286-6baeb957f778","pid":20544,"uri":"/crucible/0/regions/36fcdb41-0c7a-4daf-8916-606855849fb8/snapshots/e59028bb-7f16-4e41-80d8-a197ded34d5e/run","method":"POST","req_id":"d103b9b8-fffb-46ff-b040-60281b3a3b74","remote_addr":"[fd00:1122:3344:101::a]:34761","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25321,"response_code":"200"}