[ Jun 28 23:40:36 Enabled. ]
[ Jun 28 23:40:36 Rereading configuration. ]
[ Jun 28 23:40:39 Rereading configuration. ]
[ Jun 28 23:40:45 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_85bff804-1ecb-4fb7-82b1-ff4f554a8b8c/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_85bff804-1ecb-4fb7-82b1-ff4f554a8b8c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:46.013803849Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:46.01720823Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:46.01721884Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:46.031126753Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:46.241504371Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.655401619Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:50417"}
{"msg":"region 1dc30918-a5eb-4461-b83e-623abde34470 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.656510785Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.656892154Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions","method":"POST","req_id":"b62889af-6e31-4503-a145-5c17d442bb0c","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":680,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.657485209Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_85bff804-1ecb-4fb7-82b1-ff4f554a8b8c/crucible/regions/1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.668150244Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_85bff804-1ecb-4fb7-82b1-ff4f554a8b8c/crucible/regions/1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.668202075Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1dc30918-a5eb-4461-b83e-623abde34470\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1dc30918-a5eb-4461-b83e-623abde34470\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.828052566Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"region":"1dc30918-a5eb-4461-b83e-623abde34470","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1dc30918-a5eb-4461-b83e-623abde34470\" \"--data\" \"/data/regions/1dc30918-a5eb-4461-b83e-623abde34470\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.830178477Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"region":"1dc30918-a5eb-4461-b83e-623abde34470","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.963586916Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions","method":"POST","req_id":"aec375fc-9256-4c41-852d-6695587cab02","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":686,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.080579102Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"region":"1dc30918-a5eb-4461-b83e-623abde34470","component":"worker"}
{"msg":"region 1dc30918-a5eb-4461-b83e-623abde34470 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.080639844Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.080863669Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.081268879Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.102568468Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.130650109Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.154491789Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.155253367Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.155330429Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.15537419Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.155418691Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.169347924Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.18296218Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.258558146Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions","method":"POST","req_id":"d97c0bca-f479-4325-bded-812385762bfd","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.440090629Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions/1dc30918-a5eb-4461-b83e-623abde34470","method":"GET","req_id":"0c9d50ee-762c-4a23-a3f9-221d3005f215","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"path is \"/data/regions/1dc30918-a5eb-4461-b83e-623abde34470\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.443693345Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"dataset is oxp_85bff804-1ecb-4fb7-82b1-ff4f554a8b8c/crucible/regions/1dc30918-a5eb-4461-b83e-623abde34470","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.45351206Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"region 1dc30918-a5eb-4461-b83e-623abde34470 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.461643294Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.461847299Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.461922541Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions/1dc30918-a5eb-4461-b83e-623abde34470","method":"DELETE","req_id":"59af2066-cc98-4ddb-bc07-68f57cbddfb5","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18700,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-1dc30918-a5eb-4461-b83e-623abde34470 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.462943785Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.46438796Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions/1dc30918-a5eb-4461-b83e-623abde34470","method":"GET","req_id":"79a505a8-ffe6-493a-847f-60bf8ec6856c","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.488369303Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_85bff804-1ecb-4fb7-82b1-ff4f554a8b8c/crucible/regions/1dc30918-a5eb-4461-b83e-623abde34470\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.497945482Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"region":"1dc30918-a5eb-4461-b83e-623abde34470","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.521144196Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions/1dc30918-a5eb-4461-b83e-623abde34470","method":"GET","req_id":"810beeba-0a39-40fb-93e0-962bb3bb9ae0","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"region 1dc30918-a5eb-4461-b83e-623abde34470 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.60536173Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.605660757Z","hostname":"oxz_crucible_ffe3151d-4305-4f77-97f4-7a81da762243","pid":25156,"uri":"/crucible/0/regions/1dc30918-a5eb-4461-b83e-623abde34470","method":"GET","req_id":"499a5729-038c-4412-8470-4f205b5bd406","remote_addr":"[fd00:1122:3344:101::c]:50417","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":643,"response_code":"200"}