[ Aug 23 04:22:30 Enabled. ]
[ Aug 23 04:22:30 Rereading configuration. ]
[ Aug 23 04:22:31 Rereading configuration. ]
[ Aug 23 04:22:33 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.649670544Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.65103301Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.65104282Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.667838687Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:34.102696505Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.641505124Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52704"}
{"msg":"region 6d5464a4-a2a6-457d-a418-e373d8c42efb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.661262932Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.666004462Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.670138132Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions","method":"POST","req_id":"41a513b2-06fe-4823-b53b-4c5478bf45e0","remote_addr":"[fd00:1122:3344:101::a]:52704","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":13787,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.677748939Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.677770829Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6d5464a4-a2a6-457d-a418-e373d8c42efb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.726560708Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"6d5464a4-a2a6-457d-a418-e373d8c42efb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6d5464a4-a2a6-457d-a418-e373d8c42efb\" \"--data\" \"/data/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.72840941Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"6d5464a4-a2a6-457d-a418-e373d8c42efb","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.842545475Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"6d5464a4-a2a6-457d-a418-e373d8c42efb","component":"worker"}
{"msg":"region 6d5464a4-a2a6-457d-a418-e373d8c42efb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.842597087Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.8426837Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.843101054Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.846599942Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.859794568Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.863094999Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.869961141Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.871937028Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.87199794Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.872047711Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.87673455Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:29.881430628Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.009859465Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions","method":"POST","req_id":"f46dc370-0847-4b4d-bf06-80a393ad38ab","remote_addr":"[fd00:1122:3344:101::a]:52704","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.705506952Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","method":"GET","req_id":"5c8a2052-7fec-4280-8424-b9bee5838dd1","remote_addr":"[fd00:1122:3344:101::a]:52704","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":658,"response_code":"200"}
{"msg":"path is \"/data/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.70753643Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"dataset is oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.716920127Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"region 6d5464a4-a2a6-457d-a418-e373d8c42efb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.725527248Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.725695054Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.725776546Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","method":"DELETE","req_id":"1dff5c09-d898-4ce7-a5d3-2925f2f7e2fc","remote_addr":"[fd00:1122:3344:101::a]:52704","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18265,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6d5464a4-a2a6-457d-a418-e373d8c42efb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.726946556Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.729071397Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","method":"GET","req_id":"96828f50-f29d-455b-8e1c-2b979540d79f","remote_addr":"[fd00:1122:3344:101::a]:52704","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.731302253Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.741201837Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"6d5464a4-a2a6-457d-a418-e373d8c42efb","component":"worker"}
{"msg":"region 6d5464a4-a2a6-457d-a418-e373d8c42efb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.789118586Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:30.79368974Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions/6d5464a4-a2a6-457d-a418-e373d8c42efb","method":"GET","req_id":"9188f064-24de-4667-86ea-0aee0ac2fb79","remote_addr":"[fd00:1122:3344:101::a]:52704","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":639,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.38841802Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46923"}
{"msg":"region fc756e3a-c936-4d0c-bcc0-905366f11d04 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.388666048Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.389535247Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions","method":"POST","req_id":"d24b9140-f3da-4c0d-b6cc-636abeb0216a","remote_addr":"[fd00:1122:3344:101::a]:46923","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":913,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.389553378Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible/regions/fc756e3a-c936-4d0c-bcc0-905366f11d04","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.398586443Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1cc020bb-41b6-4914-9415-d71a097582c6/crucible/regions/fc756e3a-c936-4d0c-bcc0-905366f11d04","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.398619954Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fc756e3a-c936-4d0c-bcc0-905366f11d04\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fc756e3a-c936-4d0c-bcc0-905366f11d04\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.46361471Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"fc756e3a-c936-4d0c-bcc0-905366f11d04","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fc756e3a-c936-4d0c-bcc0-905366f11d04\" \"--data\" \"/data/regions/fc756e3a-c936-4d0c-bcc0-905366f11d04\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.465965839Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"fc756e3a-c936-4d0c-bcc0-905366f11d04","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.618955166Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"region":"fc756e3a-c936-4d0c-bcc0-905366f11d04","component":"worker"}
{"msg":"region fc756e3a-c936-4d0c-bcc0-905366f11d04 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.618992728Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.619096591Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fc756e3a-c936-4d0c-bcc0-905366f11d04","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.620577561Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fc756e3a-c936-4d0c-bcc0-905366f11d04","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.629213873Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.648599948Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fc756e3a-c936-4d0c-bcc0-905366f11d04","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.657344253Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fc756e3a-c936-4d0c-bcc0-905366f11d04","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.657644933Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.657713926Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.657765187Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.657814349Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.665829849Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.678618452Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:41.68034779Z","hostname":"oxz_crucible_2d16baf4-b5fb-4717-9984-b204d421a11c","pid":21087,"uri":"/crucible/0/regions","method":"POST","req_id":"e2d9d273-2f58-4af0-bb7f-01cf81bcde60","remote_addr":"[fd00:1122:3344:101::a]:46923","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}