[ Apr 29 22:38:25 Enabled. ]
[ Apr 29 22:38:25 Rereading configuration. ]
[ Apr 29 22:38:27 Rereading configuration. ]
[ Apr 29 22:38:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.639835328Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.644715139Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.64473338Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.673064835Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.840367056Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.96578151Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:56695"}
{"msg":"region f42c4132-6c98-4b41-8bf1-761d19d0fded state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.968256597Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.969074345Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.969089576Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions","method":"POST","req_id":"a9d40e02-2580-4575-9c4e-c4f97dad7b2e","remote_addr":"[fd00:1122:3344:101::c]:56695","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":1706,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crucible/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.978777626Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crucible/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.978814847Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f42c4132-6c98-4b41-8bf1-761d19d0fded\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.031877256Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"region":"f42c4132-6c98-4b41-8bf1-761d19d0fded","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f42c4132-6c98-4b41-8bf1-761d19d0fded\" \"--data\" \"/data/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.031962978Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"region":"f42c4132-6c98-4b41-8bf1-761d19d0fded","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.144790368Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"region":"f42c4132-6c98-4b41-8bf1-761d19d0fded","component":"worker"}
{"msg":"region f42c4132-6c98-4b41-8bf1-761d19d0fded state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.14485165Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.144947572Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.145392652Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.14882102Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.161076459Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164162909Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164742993Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164838465Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164884186Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164924967Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.168748974Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.173770558Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.269485029Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions","method":"POST","req_id":"f12f2779-44fc-4135-977d-52ac80df3182","remote_addr":"[fd00:1122:3344:101::c]:56695","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.139388015Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:47264"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.139475757Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","method":"GET","req_id":"a7f97794-6a6c-46f4-aa6e-5cf63fb87457","remote_addr":"[fd00:1122:3344:101::c]:47264","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"path is \"/data/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.139639421Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"dataset is oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crucible/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.148880591Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"region f42c4132-6c98-4b41-8bf1-761d19d0fded state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.157459387Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.15760084Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.157725153Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","method":"DELETE","req_id":"7602f843-f430-4e34-900b-2a310cd520dd","remote_addr":"[fd00:1122:3344:101::c]:47264","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18114,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.157909357Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","method":"GET","req_id":"bb5d5697-594a-43fe-9af6-75f283d9cb51","remote_addr":"[fd00:1122:3344:101::c]:47264","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-f42c4132-6c98-4b41-8bf1-761d19d0fded (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.158720325Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.162409889Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crucible/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.172595992Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"region":"f42c4132-6c98-4b41-8bf1-761d19d0fded","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.211243362Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","method":"GET","req_id":"1008ca01-32fb-4d09-9f8f-4947a699c3af","remote_addr":"[fd00:1122:3344:101::c]:47264","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"region f42c4132-6c98-4b41-8bf1-761d19d0fded state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.229321904Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:53.356428259Z","hostname":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","pid":25370,"uri":"/crucible/0/regions/f42c4132-6c98-4b41-8bf1-761d19d0fded","method":"GET","req_id":"9b211b23-9868-4a4c-96b6-b8675538aab4","remote_addr":"[fd00:1122:3344:101::c]:47264","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":23,"response_code":"200"}