[ Mar 13 04:51:51 Enabled. ]
[ Mar 13 04:51:51 Rereading configuration. ]
[ Mar 13 04:51:55 Rereading configuration. ]
[ Mar 13 04:52:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:04.480747458Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:04.48347935Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:04.48349749Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:04.675467856Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.327633653Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64170"}
{"msg":"region ce033bfe-9230-4d19-a9d2-912a765ad2ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.388475352Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.388764407Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"1f5422ea-1f6f-439d-94ec-ca51e717d842","remote_addr":"[fd00:1122:3344:101::a]:64170","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":1640,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.389464728Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.398872984Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.398928415Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ce033bfe-9230-4d19-a9d2-912a765ad2ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.497261034Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"ce033bfe-9230-4d19-a9d2-912a765ad2ca","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.580474758Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"dabddc76-5624-40f7-888b-8dce5299186b","remote_addr":"[fd00:1122:3344:101::a]:64170","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":920,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.669456884Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"ce033bfe-9230-4d19-a9d2-912a765ad2ca","component":"worker"}
{"msg":"region ce033bfe-9230-4d19-a9d2-912a765ad2ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.66948665Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.669717278Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.670186824Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.67896934Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.706514549Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.717283286Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.718012388Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.718087799Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.718135679Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.71817248Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.730542972Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:23.741760947Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.145715679Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"439aa6cd-12ad-49dc-85b3-21447b8fccab","remote_addr":"[fd00:1122:3344:101::a]:64170","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.487406653Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39770"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.487559385Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","method":"GET","req_id":"bde3c9d2-af6e-40a3-a941-c89df61dd4d9","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.48783897Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.496917961Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"region ce033bfe-9230-4d19-a9d2-912a765ad2ca state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.505047577Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.505263651Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.505321762Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","method":"DELETE","req_id":"00073cc0-1d22-442d-b5c2-65b7a67551cf","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":17507,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.505508995Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","method":"GET","req_id":"144f2c14-5ed4-434d-8c41-ed5547907441","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-ce033bfe-9230-4d19-a9d2-912a765ad2ca (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.506606982Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.513312026Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.523276441Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"ce033bfe-9230-4d19-a9d2-912a765ad2ca","component":"worker"}
{"msg":"region ce033bfe-9230-4d19-a9d2-912a765ad2ca state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.591230358Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:24.640415713Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/ce033bfe-9230-4d19-a9d2-912a765ad2ca","method":"GET","req_id":"f9300cde-dd08-4175-8120-e3832d82f7d1","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":579,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.464701104Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54517"}
{"msg":"region dedd7bc4-90c2-48f0-96ae-93981d75768b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.465581348Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.46575747Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"edab5b29-276e-4d50-bd9c-af440679815f","remote_addr":"[fd00:1122:3344:101::a]:54517","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":1006,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.465796861Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.475281578Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.47534926Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dedd7bc4-90c2-48f0-96ae-93981d75768b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.560827519Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"dedd7bc4-90c2-48f0-96ae-93981d75768b","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.670256881Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"dedd7bc4-90c2-48f0-96ae-93981d75768b","component":"worker"}
{"msg":"region dedd7bc4-90c2-48f0-96ae-93981d75768b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.670298811Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.670525305Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.672023178Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.675552663Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.689840766Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.699672818Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.699913122Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.699968903Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.700011064Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.700048644Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.706957642Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.712570069Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:52:26.820316395Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"f0fb31f5-d8eb-45c7-b00b-ffa45adc1289","remote_addr":"[fd00:1122:3344:101::a]:54517","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.636777485Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35194"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.637346934Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b","method":"GET","req_id":"f9c0daf1-95bd-43df-a22b-05ca548b4f0d","remote_addr":"[fd00:1122:3344:101::a]:35194","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":510,"response_code":"200"}
{"msg":"path is \"/data/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.637509636Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.646155251Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"snapshot is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b@6c125d2f-b8d1-4fdb-bccf-20213af38f6d","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.654346298Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b\", \"6c125d2f-b8d1-4fdb-bccf-20213af38f6d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.654378568Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.66539626Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b/snapshots/6c125d2f-b8d1-4fdb-bccf-20213af38f6d","method":"GET","req_id":"79d3b34c-a64f-401d-8158-1d2a45f0e1b5","remote_addr":"[fd00:1122:3344:101::a]:35194","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27895,"response_code":"200"}
{"msg":"path is \"/data/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.665572793Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.673033969Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"snapshot is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b@6c125d2f-b8d1-4fdb-bccf-20213af38f6d","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.680963372Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b\", \"6c125d2f-b8d1-4fdb-bccf-20213af38f6d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.680984122Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot dedd7bc4-90c2-48f0-96ae-93981d75768b-6c125d2f-b8d1-4fdb-bccf-20213af38f6d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.691669939Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"applying SMF actions for region dedd7bc4-90c2-48f0-96ae-93981d75768b running snapshot 6c125d2f-b8d1-4fdb-bccf-20213af38f6d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.692248908Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.692293218Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b/snapshots/6c125d2f-b8d1-4fdb-bccf-20213af38f6d/run","method":"POST","req_id":"bc6b80ea-0a83-4f9d-af6e-39a510903be7","remote_addr":"[fd00:1122:3344:101::a]:35194","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26750,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-dedd7bc4-90c2-48f0-96ae-93981d75768b-6c125d2f-b8d1-4fdb-bccf-20213af38f6d","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.700738789Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-dedd7bc4-90c2-48f0-96ae-93981d75768b-6c125d2f-b8d1-4fdb-bccf-20213af38f6d","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.706344127Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-dedd7bc4-90c2-48f0-96ae-93981d75768b-6c125d2f-b8d1-4fdb-bccf-20213af38f6d","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.71749245Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dedd7bc4-90c2-48f0-96ae-93981d75768b/.zfs/snapshot/6c125d2f-b8d1-4fdb-bccf-20213af38f6d","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.717691183Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.717740194Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.717778274Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.717814955Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.717859766Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.720952004Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.724846904Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"region dedd7bc4-90c2-48f0-96ae-93981d75768b running snapshot 6c125d2f-b8d1-4fdb-bccf-20213af38f6d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:45.724854245Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.508561839Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57220"}
{"msg":"region 9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.567949303Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.568131466Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"a9563935-e866-43ab-ad3b-29acc27f4240","remote_addr":"[fd00:1122:3344:101::a]:57220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":285,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.568154256Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.578079231Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.578117281Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.652889434Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.712642033Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"562ee083-7377-4f8f-8e9e-111a433f077d","remote_addr":"[fd00:1122:3344:101::a]:57220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.744001371Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","component":"worker"}
{"msg":"region 9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.744042202Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.744158224Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.745353872Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.748332719Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.755349998Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.758000079Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.758270353Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.758317494Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.758360625Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.758401505Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.761677806Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:48.771376387Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:57:49.431960021Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions","method":"POST","req_id":"1d76e6ce-91e3-46c9-8f5e-af6f35a3a465","remote_addr":"[fd00:1122:3344:101::a]:57220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.396266944Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61193"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.396347985Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","method":"GET","req_id":"3ddf5df6-6927-484f-85a4-cb9ae9278ad0","remote_addr":"[fd00:1122:3344:101::a]:61193","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":43,"response_code":"200"}
{"msg":"path is \"/data/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.396509558Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.40759053Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"region 9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.415662096Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.415877539Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.41593728Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","method":"DELETE","req_id":"5d43c8f1-b1d1-4a06-8017-df6e6c1478b7","remote_addr":"[fd00:1122:3344:101::a]:61193","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19449,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.416169654Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","method":"GET","req_id":"517e146b-565b-4d50-9476-43a587b15a7b","remote_addr":"[fd00:1122:3344:101::a]:61193","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.417558815Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.424238479Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.433879899Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"region":"9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","component":"worker"}
{"msg":"region 9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.479240674Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-13T04:58:29.523447402Z","hostname":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","pid":24350,"uri":"/crucible/0/regions/9c6a2f6e-2ba9-4cf3-88b4-656bae85bee6","method":"GET","req_id":"0281cf0d-bf0e-463a-8da1-eefe8879618e","remote_addr":"[fd00:1122:3344:101::a]:61193","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":52,"response_code":"200"}