[ Jul 17 23:51:32 Enabled. ]
[ Jul 17 23:51:32 Rereading configuration. ]
[ Jul 17 23:51:34 Rereading configuration. ]
[ Jul 17 23:51:39 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:51:39.845868213Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:51:39.849135051Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:51:39.849142814Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:51:39.864683952Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:51:39.967999032Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.265196618Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58844"}
{"msg":"region 30e40901-dcdc-4694-9784-bacb2e0f6993 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.284050118Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.289546703Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions","method":"POST","req_id":"21580af1-e317-42c0-a30d-64127347c1ac","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16970,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.29196549Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.30236167Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.302419451Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"30e40901-dcdc-4694-9784-bacb2e0f6993\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/30e40901-dcdc-4694-9784-bacb2e0f6993\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.36539596Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"30e40901-dcdc-4694-9784-bacb2e0f6993","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"30e40901-dcdc-4694-9784-bacb2e0f6993\" \"--data\" \"/data/regions/30e40901-dcdc-4694-9784-bacb2e0f6993\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.367402051Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"30e40901-dcdc-4694-9784-bacb2e0f6993","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.437636422Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions","method":"POST","req_id":"8bd69c9e-a952-4776-9d05-f084991d9b3a","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.662183599Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"30e40901-dcdc-4694-9784-bacb2e0f6993","component":"worker"}
{"msg":"region 30e40901-dcdc-4694-9784-bacb2e0f6993 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.6622598Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.662488364Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.663186485Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.6687393Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.684670965Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.688309361Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.689137494Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.689212295Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.689257376Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.689299117Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.693476361Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.707786196Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:17.978695832Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions","method":"POST","req_id":"3c5ab0a7-aedd-4d1b-8ae2-52443018a3e2","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.432159643Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","method":"GET","req_id":"b1c42ed4-b826-4a64-80fe-d68e81689037","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"path is \"/data/regions/30e40901-dcdc-4694-9784-bacb2e0f6993\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.435209659Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"dataset is oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.447384177Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"region 30e40901-dcdc-4694-9784-bacb2e0f6993 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.457900239Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.458152723Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.458212734Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","method":"DELETE","req_id":"28cad142-4420-4cbd-a9e3-ad709e3c5c0f","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23024,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-30e40901-dcdc-4694-9784-bacb2e0f6993 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.459179208Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.460654971Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","method":"GET","req_id":"b653bd43-54d3-475b-a540-168fec918979","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.463374683Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible/regions/30e40901-dcdc-4694-9784-bacb2e0f6993\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.475873625Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"30e40901-dcdc-4694-9784-bacb2e0f6993","component":"worker"}
{"msg":"region 30e40901-dcdc-4694-9784-bacb2e0f6993 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.547808843Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:18.607281968Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions/30e40901-dcdc-4694-9784-bacb2e0f6993","method":"GET","req_id":"b9f7965b-14b0-4741-84b0-9b6a572b7d9b","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"region 085462c6-2223-424b-b07c-74bde5266895 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.777341676Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.777561499Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions","method":"POST","req_id":"bc0b8a3d-8fd4-4c20-8cb4-3115649ebd7a","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":289,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.777625871Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible/regions/085462c6-2223-424b-b07c-74bde5266895","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.787382361Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_646c0155-c143-4b02-8fbc-abd97d5f6dc6/crucible/regions/085462c6-2223-424b-b07c-74bde5266895","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.787411481Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"085462c6-2223-424b-b07c-74bde5266895\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/085462c6-2223-424b-b07c-74bde5266895\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.863839928Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"085462c6-2223-424b-b07c-74bde5266895","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"085462c6-2223-424b-b07c-74bde5266895\" \"--data\" \"/data/regions/085462c6-2223-424b-b07c-74bde5266895\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.865887659Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"085462c6-2223-424b-b07c-74bde5266895","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.939917299Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"region":"085462c6-2223-424b-b07c-74bde5266895","component":"worker"}
{"msg":"region 085462c6-2223-424b-b07c-74bde5266895 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.93998937Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.940210294Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-085462c6-2223-424b-b07c-74bde5266895","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.941560344Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.942930635Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"uri":"/crucible/0/regions","method":"POST","req_id":"8a2071e2-6752-42e9-9119-619b1286b709","remote_addr":"[fd00:1122:3344:101::a]:58844","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-085462c6-2223-424b-b07c-74bde5266895","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.944922816Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.953031191Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-085462c6-2223-424b-b07c-74bde5266895","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.956549065Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/085462c6-2223-424b-b07c-74bde5266895","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.95684025Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.95689138Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.956941721Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.956989672Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.961102315Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T23:52:20.966057281Z","hostname":"oxz_crucible_a8f436c4-3f32-49e2-848b-951bd1d28b96","pid":24486,"component":"worker"}