[ Jan 28 23:05:29 Enabled. ]
[ Jan 28 23:05:29 Rereading configuration. ]
[ Jan 28 23:05:33 Rereading configuration. ]
[ Jan 28 23:05:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ff6949fe-2851-437a-b6e9-cb91f2325920/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ff6949fe-2851-437a-b6e9-cb91f2325920/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:05:43.992706807Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:05:43.993563411Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:05:43.993572731Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:05:44.009210582Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:05:44.139967606Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.291848399Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:65274"}
{"msg":"region 68a4bbec-1c99-4fab-b4f4-9435d497cf1c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.333340982Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.340653014Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions","method":"POST","req_id":"f610042b-f167-418d-9081-53c4b753b153","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20833,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.340678465Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ff6949fe-2851-437a-b6e9-cb91f2325920/crucible/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.350286935Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ff6949fe-2851-437a-b6e9-cb91f2325920/crucible/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.352607484Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"68a4bbec-1c99-4fab-b4f4-9435d497cf1c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.435511168Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"region":"68a4bbec-1c99-4fab-b4f4-9435d497cf1c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"68a4bbec-1c99-4fab-b4f4-9435d497cf1c\" \"--data\" \"/data/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.437780476Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"region":"68a4bbec-1c99-4fab-b4f4-9435d497cf1c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.558223007Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions","method":"POST","req_id":"d58f4658-c0b9-4e1f-b0e5-e2a6f97cdc90","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.750335395Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"region":"68a4bbec-1c99-4fab-b4f4-9435d497cf1c","component":"worker"}
{"msg":"region 68a4bbec-1c99-4fab-b4f4-9435d497cf1c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.750381366Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.750485208Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.750935215Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.759054611Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.774954756Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.791377781Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.791897249Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.79431873Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.794376571Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.794414861Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.801141264Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:47.805946424Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.145425412Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions","method":"POST","req_id":"7bb436b7-1cbe-4024-a658-41cfc09815ad","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.64056195Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","method":"GET","req_id":"2fa003ee-37e0-4b61-b17e-cf77328fd456","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.643350967Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"dataset is oxp_ff6949fe-2851-437a-b6e9-cb91f2325920/crucible/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.654200568Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"region 68a4bbec-1c99-4fab-b4f4-9435d497cf1c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.663281679Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.663432222Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-68a4bbec-1c99-4fab-b4f4-9435d497cf1c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.664552581Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.665423975Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","method":"DELETE","req_id":"d5720dcc-9956-439e-80c4-3877ff6928d6","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22116,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.668868013Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","method":"GET","req_id":"85560135-217c-4110-be32-2f7cbd581d84","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.674153331Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ff6949fe-2851-437a-b6e9-cb91f2325920/crucible/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.686262723Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"region":"68a4bbec-1c99-4fab-b4f4-9435d497cf1c","component":"worker"}
{"msg":"region 68a4bbec-1c99-4fab-b4f4-9435d497cf1c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.756509746Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-28T23:06:48.815309798Z","hostname":"oxz_crucible_66ab6032-c613-4d07-ba88-c2c852d0f0e4","pid":22456,"uri":"/crucible/0/regions/68a4bbec-1c99-4fab-b4f4-9435d497cf1c","method":"GET","req_id":"eb3b3bf9-d18b-4c79-943b-1036d6781d05","remote_addr":"[fd00:1122:3344:101::c]:65274","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":80,"response_code":"200"}