[ Mar  1 23:17:32 Enabled. ]
[ Mar  1 23:17:32 Rereading configuration. ]
[ Mar  1 23:17:37 Rereading configuration. ]
[ Mar  1 23:17:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:43.875258536Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:43.876029865Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:43.876038965Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:44.289265407Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.356623491Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61425"}
{"msg":"region 77115e99-3a1f-406f-bb6f-eaa5f35c2a84 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.366566345Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.367463127Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"a0aa53f6-19ca-40da-8f12-27e06a125f6b","remote_addr":"[fd00:1122:3344:101::c]:61425","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":5828,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.368087372Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.376897987Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.376963019Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"77115e99-3a1f-406f-bb6f-eaa5f35c2a84\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.440869194Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"region":"77115e99-3a1f-406f-bb6f-eaa5f35c2a84","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.573281617Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"region":"77115e99-3a1f-406f-bb6f-eaa5f35c2a84","component":"worker"}
{"msg":"region 77115e99-3a1f-406f-bb6f-eaa5f35c2a84 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.573330701Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.573558022Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.573939631Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.581072538Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.591601336Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.596116602Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.596735162Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.596816984Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.596863995Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.596910096Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.602503093Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.608346496Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:24.72973544Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"b3454ec5-6982-467a-98e4-7e9e2f470b04","remote_addr":"[fd00:1122:3344:101::c]:61425","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.661819262Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:46348"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.661952605Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","method":"GET","req_id":"8fd06b0c-7da5-4ba7-b708-2973c71708f0","remote_addr":"[fd00:1122:3344:101::c]:46348","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":110,"response_code":"200"}
{"msg":"path is \"/data/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.662078388Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.671084819Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"region 77115e99-3a1f-406f-bb6f-eaa5f35c2a84 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.679243759Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.679411513Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.679481534Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","method":"DELETE","req_id":"40388d7a-8363-493e-bec4-0371ba9d648b","remote_addr":"[fd00:1122:3344:101::c]:46348","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":17421,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.679613768Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","method":"GET","req_id":"d1c0b373-64b7-4dc2-aa6f-2f8bfb95c7ba","remote_addr":"[fd00:1122:3344:101::c]:46348","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-77115e99-3a1f-406f-bb6f-eaa5f35c2a84 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.680624182Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.684330803Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.693233261Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"region":"77115e99-3a1f-406f-bb6f-eaa5f35c2a84","component":"worker"}
{"msg":"region 77115e99-3a1f-406f-bb6f-eaa5f35c2a84 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.745818429Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:18:25.794893401Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions/77115e99-3a1f-406f-bb6f-eaa5f35c2a84","method":"GET","req_id":"d7a614c7-9af9-4ce0-a4ba-bfa6b20fc08b","remote_addr":"[fd00:1122:3344:101::c]:46348","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.822619492Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39214"}
{"msg":"region 4191742d-78a6-4330-b3c4-72d2bb7984f4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.822755786Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.823854092Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"8c855b23-dbeb-43fe-833b-c7c4ec7f6679","remote_addr":"[fd00:1122:3344:101::a]:39214","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1187,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.823874513Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/4191742d-78a6-4330-b3c4-72d2bb7984f4","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.833004866Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/4191742d-78a6-4330-b3c4-72d2bb7984f4","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.833030267Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4191742d-78a6-4330-b3c4-72d2bb7984f4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/4191742d-78a6-4330-b3c4-72d2bb7984f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:29.922928219Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"region":"4191742d-78a6-4330-b3c4-72d2bb7984f4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.168599418Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"cb784e8f-e5af-4951-b4fa-e00962a8c2aa","remote_addr":"[fd00:1122:3344:101::a]:39214","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.241987475Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"region":"4191742d-78a6-4330-b3c4-72d2bb7984f4","component":"worker"}
{"msg":"region 4191742d-78a6-4330-b3c4-72d2bb7984f4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.242011166Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.242124539Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4191742d-78a6-4330-b3c4-72d2bb7984f4","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.243458361Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4191742d-78a6-4330-b3c4-72d2bb7984f4","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.261588465Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.292101673Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4191742d-78a6-4330-b3c4-72d2bb7984f4","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.305163583Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4191742d-78a6-4330-b3c4-72d2bb7984f4","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.305401959Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.30545891Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.305508341Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.305556893Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.316301076Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.329557461Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:30.551077686Z","hostname":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"fe93c7ea-8065-4a50-aa7f-303d6406dae7","remote_addr":"[fd00:1122:3344:101::a]:39214","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"}