[ Jul 18 19:14:56 Enabled. ]
[ Jul 18 19:14:56 Rereading configuration. ]
[ Jul 18 19:14:59 Rereading configuration. ]
[ Jul 18 19:15:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.349846084Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.353153535Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.353163407Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.368634742Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.507827205Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:30.999313314Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:65175"}
{"msg":"region 3e77c582-6607-416d-904d-d725fc796064 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.024129796Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.028674852Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions","method":"POST","req_id":"ad7fb912-b864-41d7-a528-df27084d0f85","remote_addr":"[fd00:1122:3344:101::a]:65175","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19023,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.031258265Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.040421026Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.040506748Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3e77c582-6607-416d-904d-d725fc796064\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3e77c582-6607-416d-904d-d725fc796064\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.119472722Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"region":"3e77c582-6607-416d-904d-d725fc796064","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3e77c582-6607-416d-904d-d725fc796064\" \"--data\" \"/data/regions/3e77c582-6607-416d-904d-d725fc796064\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.12129552Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"region":"3e77c582-6607-416d-904d-d725fc796064","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.194074559Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions","method":"POST","req_id":"ac27d7a9-9600-45d9-b2aa-008483fb1d5d","remote_addr":"[fd00:1122:3344:101::a]:65175","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.439482202Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"region":"3e77c582-6607-416d-904d-d725fc796064","component":"worker"}
{"msg":"region 3e77c582-6607-416d-904d-d725fc796064 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.439553113Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.439811157Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.440250034Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.445603206Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.460351783Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463242187Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463810186Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463872227Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463910818Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463952858Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.468155753Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.481731852Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.814165232Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions","method":"POST","req_id":"f32d1bf3-f930-42c8-acf7-871335ba9296","remote_addr":"[fd00:1122:3344:101::a]:65175","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.941222118Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:65404"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.945743728Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions/3e77c582-6607-416d-904d-d725fc796064","method":"GET","req_id":"88c304ec-3e10-430b-a409-63c406d4aeea","remote_addr":"[fd00:1122:3344:101::a]:65404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":4448,"response_code":"200"}
{"msg":"path is \"/data/regions/3e77c582-6607-416d-904d-d725fc796064\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.948689073Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"dataset is oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.957869504Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"snapshot is oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064@116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.967006684Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064\", \"116a239b-954f-4639-80ac-bb95bc129003\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.967038295Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.978273648Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions/3e77c582-6607-416d-904d-d725fc796064/snapshots/116a239b-954f-4639-80ac-bb95bc129003","method":"GET","req_id":"4536448e-70e7-4248-b332-cd2dca6ea2a8","remote_addr":"[fd00:1122:3344:101::a]:65404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29584,"response_code":"200"}
{"msg":"path is \"/data/regions/3e77c582-6607-416d-904d-d725fc796064\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.982084786Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"dataset is oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.99009129Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"snapshot is oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064@116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.998743143Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064\", \"116a239b-954f-4639-80ac-bb95bc129003\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.998774153Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 3e77c582-6607-416d-904d-d725fc796064-116a239b-954f-4639-80ac-bb95bc129003 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.008093286Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"applying SMF actions for region 3e77c582-6607-416d-904d-d725fc796064 running snapshot 116a239b-954f-4639-80ac-bb95bc129003 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.008721626Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.008762826Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions/3e77c582-6607-416d-904d-d725fc796064/snapshots/116a239b-954f-4639-80ac-bb95bc129003/run","method":"POST","req_id":"0d5f81ab-517d-4639-9408-fe7d90eb6e69","remote_addr":"[fd00:1122:3344:101::a]:65404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26701,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-3e77c582-6607-416d-904d-d725fc796064-116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.026608381Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3e77c582-6607-416d-904d-d725fc796064-116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.04277358Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3e77c582-6607-416d-904d-d725fc796064-116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.062011355Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3e77c582-6607-416d-904d-d725fc796064/.zfs/snapshot/116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.062220968Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.062269569Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.06230666Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.06234252Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.062388161Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.067798774Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.072422005Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"worker"}
{"msg":"region 3e77c582-6607-416d-904d-d725fc796064 running snapshot 116a239b-954f-4639-80ac-bb95bc129003 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.072439096Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"path is \"/data/regions/3e77c582-6607-416d-904d-d725fc796064\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.225049562Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"dataset is oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.233703285Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"datafile"}
{"msg":"snapshot is oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064@116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.240686552Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8e508f4c-2811-49cd-ab16-1715133cdbf4/crucible/regions/3e77c582-6607-416d-904d-d725fc796064\", \"116a239b-954f-4639-80ac-bb95bc129003\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.240722673Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.247909683Z","hostname":"oxz_crucible_4d7bb1db-68b3-4ac9-9b25-f910f1780a34","pid":28095,"uri":"/crucible/0/regions/3e77c582-6607-416d-904d-d725fc796064/snapshots/116a239b-954f-4639-80ac-bb95bc129003/run","method":"POST","req_id":"46801475-ba3d-46d7-8279-08f0a0fef043","remote_addr":"[fd00:1122:3344:101::a]:65404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":22885,"response_code":"200"}