[ Jun 20 21:03:50 Enabled. ]
[ Jun 20 21:03:50 Rereading configuration. ]
[ Jun 20 21:03:51 Rereading configuration. ]
[ Jun 20 21:03:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.182994859Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.183344085Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.183354545Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.203871706Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.294333274Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.578453268Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:36685"}
{"msg":"region 57ade24d-dc07-464b-bfd2-408fbf16bd5f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.594442542Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.59783887Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.60193313Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"c5e59be2-c7fc-4e7f-8a2f-20347748a57e","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":13836,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.608060775Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.608114906Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"57ade24d-dc07-464b-bfd2-408fbf16bd5f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.691641025Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"57ade24d-dc07-464b-bfd2-408fbf16bd5f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"57ade24d-dc07-464b-bfd2-408fbf16bd5f\" \"--data\" \"/data/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.69366531Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"57ade24d-dc07-464b-bfd2-408fbf16bd5f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.767986891Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"d791f2f4-4fbc-4a07-9e76-c1a53d09b1ed","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.822081147Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"57ade24d-dc07-464b-bfd2-408fbf16bd5f","component":"worker"}
{"msg":"region 57ade24d-dc07-464b-bfd2-408fbf16bd5f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.822154709Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.822502084Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.822916551Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.827725564Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.841452249Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.854607944Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.855089622Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.855182734Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.855221914Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.855271055Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.859525058Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:16.865197665Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.313615499Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"fca0b0ed-9b30-42cb-9ee2-8c437b8435e8","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.733190568Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","method":"GET","req_id":"d2c4125b-f987-4762-a3ed-4c38e9dd1f86","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":517,"response_code":"200"}
{"msg":"path is \"/data/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.735433587Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"dataset is oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.744804917Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"region 57ade24d-dc07-464b-bfd2-408fbf16bd5f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.75313499Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.753357513Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-57ade24d-dc07-464b-bfd2-408fbf16bd5f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.754503203Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.755442899Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","method":"DELETE","req_id":"e95313ef-3525-4c3c-936e-eb3c678ea98e","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20057,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.758718835Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","method":"GET","req_id":"08a3cdaa-19b5-42ed-bb22-8fbe122616d5","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.759912126Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.770363504Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"57ade24d-dc07-464b-bfd2-408fbf16bd5f","component":"worker"}
{"msg":"region 57ade24d-dc07-464b-bfd2-408fbf16bd5f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.837705997Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:04:17.883870847Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/57ade24d-dc07-464b-bfd2-408fbf16bd5f","method":"GET","req_id":"85cc8182-d089-41db-af1d-c04d1edfba05","remote_addr":"[fd00:1122:3344:101::c]:36685","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.162576498Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52941"}
{"msg":"region 5736af07-0dde-43d6-90dd-091400281fa4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.16270439Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.162865983Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"9091836a-98b1-414e-94b5-f173ca4d0840","remote_addr":"[fd00:1122:3344:101::a]:52941","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":201,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.162880943Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.171988488Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.172037669Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5736af07-0dde-43d6-90dd-091400281fa4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5736af07-0dde-43d6-90dd-091400281fa4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.228538216Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"5736af07-0dde-43d6-90dd-091400281fa4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5736af07-0dde-43d6-90dd-091400281fa4\" \"--data\" \"/data/regions/5736af07-0dde-43d6-90dd-091400281fa4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.230462119Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"5736af07-0dde-43d6-90dd-091400281fa4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.295575963Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"5736af07-0dde-43d6-90dd-091400281fa4","component":"worker"}
{"msg":"region 5736af07-0dde-43d6-90dd-091400281fa4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.295608844Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.295715696Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.29710461Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.299754325Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.30646531Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.308866821Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.309146796Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.309205267Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.309245077Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.309286908Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.312390581Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.316452251Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.340820938Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"aa7c6f4b-dce7-4f03-82a9-b3a463748929","remote_addr":"[fd00:1122:3344:101::a]:52941","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.84170034Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55858"}
{"msg":"region ac21495c-209d-44c2-acaa-73a3f1e91b93 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.842357781Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.842899191Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"5a74f2fd-c3e5-4710-b9e4-6208df21a623","remote_addr":"[fd00:1122:3344:101::a]:55858","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1168,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.842915511Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/ac21495c-209d-44c2-acaa-73a3f1e91b93","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.854815615Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/ac21495c-209d-44c2-acaa-73a3f1e91b93","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.854846315Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ac21495c-209d-44c2-acaa-73a3f1e91b93\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ac21495c-209d-44c2-acaa-73a3f1e91b93\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.925349142Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"ac21495c-209d-44c2-acaa-73a3f1e91b93","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ac21495c-209d-44c2-acaa-73a3f1e91b93\" \"--data\" \"/data/regions/ac21495c-209d-44c2-acaa-73a3f1e91b93\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:17.927124532Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"ac21495c-209d-44c2-acaa-73a3f1e91b93","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.024342316Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"region":"ac21495c-209d-44c2-acaa-73a3f1e91b93","component":"worker"}
{"msg":"region ac21495c-209d-44c2-acaa-73a3f1e91b93 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.024395747Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.024478148Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ac21495c-209d-44c2-acaa-73a3f1e91b93","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.030060644Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ac21495c-209d-44c2-acaa-73a3f1e91b93","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.04972919Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.065924567Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ac21495c-209d-44c2-acaa-73a3f1e91b93","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.070217961Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ac21495c-209d-44c2-acaa-73a3f1e91b93","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.070402814Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.070452335Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.070492565Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.070531486Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.076774613Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.086092812Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:18.095515744Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions","method":"POST","req_id":"b594d67a-1d33-4f72-bfe9-e62e474ccf94","remote_addr":"[fd00:1122:3344:101::a]:55858","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.146230865Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:32952"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.146321196Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/5736af07-0dde-43d6-90dd-091400281fa4","method":"GET","req_id":"93e77f5e-9331-4fe4-90a5-09cc5e0020b2","remote_addr":"[fd00:1122:3344:101::a]:32952","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/5736af07-0dde-43d6-90dd-091400281fa4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.149180555Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"dataset is oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.157839043Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"snapshot is oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4@1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.166815137Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4\", \"1754a985-200c-4441-a679-cc9cc7979745\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.166848997Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.180157695Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/5736af07-0dde-43d6-90dd-091400281fa4/snapshots/1754a985-200c-4441-a679-cc9cc7979745","method":"GET","req_id":"8db09d4a-1e2b-4291-8ba3-a92c45400ced","remote_addr":"[fd00:1122:3344:101::a]:32952","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":33655,"response_code":"200"}
{"msg":"path is \"/data/regions/5736af07-0dde-43d6-90dd-091400281fa4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.180361498Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"dataset is oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.189295321Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"snapshot is oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4@1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.198637361Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible/regions/5736af07-0dde-43d6-90dd-091400281fa4\", \"1754a985-200c-4441-a679-cc9cc7979745\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.198675232Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5736af07-0dde-43d6-90dd-091400281fa4-1754a985-200c-4441-a679-cc9cc7979745 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.207980031Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}
{"msg":"applying SMF actions for region 5736af07-0dde-43d6-90dd-091400281fa4 running snapshot 1754a985-200c-4441-a679-cc9cc7979745 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.208165594Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.208201085Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"uri":"/crucible/0/regions/5736af07-0dde-43d6-90dd-091400281fa4/snapshots/1754a985-200c-4441-a679-cc9cc7979745/run","method":"POST","req_id":"34fb3299-42ff-44d8-80ec-7acc6bbc37db","remote_addr":"[fd00:1122:3344:101::a]:32952","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27871,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-5736af07-0dde-43d6-90dd-091400281fa4-1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.211086884Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5736af07-0dde-43d6-90dd-091400281fa4-1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.21785623Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5736af07-0dde-43d6-90dd-091400281fa4-1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.237679989Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5736af07-0dde-43d6-90dd-091400281fa4/.zfs/snapshot/1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.237973635Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.238039385Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.238084106Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.238130617Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.238188968Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.242759636Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.249029344Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"worker"}
{"msg":"region 5736af07-0dde-43d6-90dd-091400281fa4 running snapshot 1754a985-200c-4441-a679-cc9cc7979745 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.249057684Z","hostname":"oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc","pid":22614,"component":"datafile"}