[ Jul  3 17:22:24 Enabled. ]
[ Jul  3 17:22:24 Rereading configuration. ]
[ Jul  3 17:22:26 Rereading configuration. ]
[ Jul  3 17:22:30 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:30.997270092Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.002693914Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.002709434Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.015978952Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.310448079Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.863302523Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:41313"}
{"msg":"region a385d8e5-f3dc-486b-9f7f-7f48297893f5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.866290461Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.868767336Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions","method":"POST","req_id":"22ee22c9-6fae-4c78-b8a2-8c904e0af184","remote_addr":"[fd00:1122:3344:101::b]:41313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2800,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.868787807Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.878412549Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.87847253Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a385d8e5-f3dc-486b-9f7f-7f48297893f5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.946723746Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"a385d8e5-f3dc-486b-9f7f-7f48297893f5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a385d8e5-f3dc-486b-9f7f-7f48297893f5\" \"--data\" \"/data/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:48.948570645Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"a385d8e5-f3dc-486b-9f7f-7f48297893f5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.090365476Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"a385d8e5-f3dc-486b-9f7f-7f48297893f5","component":"worker"}
{"msg":"region a385d8e5-f3dc-486b-9f7f-7f48297893f5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.090453019Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.090647334Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.091092615Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.095679745Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions","method":"POST","req_id":"02c4a2ac-013a-438b-afd1-7a763cda4e62","remote_addr":"[fd00:1122:3344:101::b]:41313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.097186365Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.106231132Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.111047028Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.111781647Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.111867619Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.111924001Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.111968462Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.11687015Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.121652845Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.831704041Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","method":"GET","req_id":"12f415bb-5412-4235-9861-54337a7858bc","remote_addr":"[fd00:1122:3344:101::b]:41313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.833841606Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"dataset is oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.843398636Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"region a385d8e5-f3dc-486b-9f7f-7f48297893f5 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.851908839Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.852113005Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.852184057Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","method":"DELETE","req_id":"c066fdd5-d6da-4ab8-9c52-e2268ee05825","remote_addr":"[fd00:1122:3344:101::b]:41313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18391,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-a385d8e5-f3dc-486b-9f7f-7f48297893f5 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.853180283Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.855338439Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","method":"GET","req_id":"e6d5b792-ef9d-4b42-9383-fbef225ac24c","remote_addr":"[fd00:1122:3344:101::b]:41313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.857400773Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.866886181Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"a385d8e5-f3dc-486b-9f7f-7f48297893f5","component":"worker"}
{"msg":"region a385d8e5-f3dc-486b-9f7f-7f48297893f5 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.921993414Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:49.959585408Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/a385d8e5-f3dc-486b-9f7f-7f48297893f5","method":"GET","req_id":"ab93bd1c-2725-413e-a6d0-7e3ca184d285","remote_addr":"[fd00:1122:3344:101::b]:41313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.410865443Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60170"}
{"msg":"region 3d473b97-1350-4629-a763-1ea7eba08f26 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.410965216Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.411189281Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.411204822Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions","method":"POST","req_id":"939dc2c8-d062-4d85-ac18-9af53961054a","remote_addr":"[fd00:1122:3344:101::a]:60170","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":263,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.420094424Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.420123535Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3d473b97-1350-4629-a763-1ea7eba08f26\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3d473b97-1350-4629-a763-1ea7eba08f26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.473826281Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"3d473b97-1350-4629-a763-1ea7eba08f26","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3d473b97-1350-4629-a763-1ea7eba08f26\" \"--data\" \"/data/regions/3d473b97-1350-4629-a763-1ea7eba08f26\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.47569759Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"3d473b97-1350-4629-a763-1ea7eba08f26","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.567666677Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"3d473b97-1350-4629-a763-1ea7eba08f26","component":"worker"}
{"msg":"region 3d473b97-1350-4629-a763-1ea7eba08f26 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.567701118Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.567848972Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.56928991Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.575342328Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.589967581Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.599050939Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.599347996Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.599406298Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.599446719Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.59948842Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.607376926Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.614437621Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.767854047Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions","method":"POST","req_id":"6f153461-d506-470f-8e6d-c3961e59935c","remote_addr":"[fd00:1122:3344:101::a]:60170","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.328213486Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63182"}
{"msg":"region 63055ca4-32d9-4381-9da5-bc0d63fcb885 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.32836238Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.329199191Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.329210082Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions","method":"POST","req_id":"4a722e04-2cda-49cf-859d-e3e1ffe6bcc7","remote_addr":"[fd00:1122:3344:101::a]:63182","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":913,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/63055ca4-32d9-4381-9da5-bc0d63fcb885","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.342169621Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/63055ca4-32d9-4381-9da5-bc0d63fcb885","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.342199262Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"63055ca4-32d9-4381-9da5-bc0d63fcb885\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/63055ca4-32d9-4381-9da5-bc0d63fcb885\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.402260854Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"63055ca4-32d9-4381-9da5-bc0d63fcb885","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"63055ca4-32d9-4381-9da5-bc0d63fcb885\" \"--data\" \"/data/regions/63055ca4-32d9-4381-9da5-bc0d63fcb885\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.404274756Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"63055ca4-32d9-4381-9da5-bc0d63fcb885","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.507091358Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"region":"63055ca4-32d9-4381-9da5-bc0d63fcb885","component":"worker"}
{"msg":"region 63055ca4-32d9-4381-9da5-bc0d63fcb885 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.507135789Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.507243922Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-63055ca4-32d9-4381-9da5-bc0d63fcb885","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.514337677Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-63055ca4-32d9-4381-9da5-bc0d63fcb885","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.522710147Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.532251426Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-63055ca4-32d9-4381-9da5-bc0d63fcb885","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.534708571Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/63055ca4-32d9-4381-9da5-bc0d63fcb885","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.534970718Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.535027909Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.53506793Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.535105811Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.538641244Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.543113491Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.694448922Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions","method":"POST","req_id":"3f81e07b-16c3-4122-bff4-ef6d3c29572a","remote_addr":"[fd00:1122:3344:101::a]:63182","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":132,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.498471637Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43178"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.49855434Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/3d473b97-1350-4629-a763-1ea7eba08f26","method":"GET","req_id":"d98a2253-8352-48ea-9e5a-b2465ba2b8ed","remote_addr":"[fd00:1122:3344:101::a]:43178","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/3d473b97-1350-4629-a763-1ea7eba08f26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.498738464Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"dataset is oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.507008081Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"snapshot is oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26@f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.516030997Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26\", \"f21c53cc-7f60-4c98-85eb-88fe670d1630\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.516070358Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.529018237Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/3d473b97-1350-4629-a763-1ea7eba08f26/snapshots/f21c53cc-7f60-4c98-85eb-88fe670d1630","method":"GET","req_id":"384b9bd6-ed49-4cbb-a0d7-488a08cdd860","remote_addr":"[fd00:1122:3344:101::a]:43178","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":30286,"response_code":"200"}
{"msg":"path is \"/data/regions/3d473b97-1350-4629-a763-1ea7eba08f26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.529191652Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"dataset is oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.540469087Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"snapshot is oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26@f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.553388965Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b1a69700-f4f0-431b-a2e4-01730c4dc7ca/crucible/regions/3d473b97-1350-4629-a763-1ea7eba08f26\", \"f21c53cc-7f60-4c98-85eb-88fe670d1630\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.553440326Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 3d473b97-1350-4629-a763-1ea7eba08f26-f21c53cc-7f60-4c98-85eb-88fe670d1630 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.570432671Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}
{"msg":"applying SMF actions for region 3d473b97-1350-4629-a763-1ea7eba08f26 running snapshot f21c53cc-7f60-4c98-85eb-88fe670d1630 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.570665837Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.570700198Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"uri":"/crucible/0/regions/3d473b97-1350-4629-a763-1ea7eba08f26/snapshots/f21c53cc-7f60-4c98-85eb-88fe670d1630/run","method":"POST","req_id":"41aaaa63-f3d4-4165-aef1-ced43f68d504","remote_addr":"[fd00:1122:3344:101::a]:43178","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":41542,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-3d473b97-1350-4629-a763-1ea7eba08f26-f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.574091357Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3d473b97-1350-4629-a763-1ea7eba08f26-f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.577834395Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3d473b97-1350-4629-a763-1ea7eba08f26-f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.588226997Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d473b97-1350-4629-a763-1ea7eba08f26/.zfs/snapshot/f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.588593636Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.588656368Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.588713219Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.588757651Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.588817812Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.5929253Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.597769267Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"worker"}
{"msg":"region 3d473b97-1350-4629-a763-1ea7eba08f26 running snapshot f21c53cc-7f60-4c98-85eb-88fe670d1630 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.597786467Z","hostname":"oxz_crucible_3b81b317-ce06-4d0d-ba5f-2c8067a18f4a","pid":26449,"component":"datafile"}