[ Jul 29 22:39:44 Enabled. ]
[ Jul 29 22:39:44 Rereading configuration. ]
[ Jul 29 22:39:45 Rereading configuration. ]
[ Jul 29 22:39:47 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7d81c0ff-af0b-41a7-af78-d5743695642b/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7d81c0ff-af0b-41a7-af78-d5743695642b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:39:47.131779745Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:39:47.136307174Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:39:47.136323024Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:39:47.15825435Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:39:47.290574676Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.168626687Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61596"}
{"msg":"region de05e91b-b582-425e-840a-192a45b57166 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.187162097Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.202019037Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions","method":"POST","req_id":"2d6e001b-cda9-43cc-aa05-833066b71d38","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22990,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.202046847Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_7d81c0ff-af0b-41a7-af78-d5743695642b/crucible/regions/de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.215710065Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7d81c0ff-af0b-41a7-af78-d5743695642b/crucible/regions/de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.215739425Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"de05e91b-b582-425e-840a-192a45b57166\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/de05e91b-b582-425e-840a-192a45b57166\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.345831126Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"region":"de05e91b-b582-425e-840a-192a45b57166","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"de05e91b-b582-425e-840a-192a45b57166\" \"--data\" \"/data/regions/de05e91b-b582-425e-840a-192a45b57166\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.347718947Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"region":"de05e91b-b582-425e-840a-192a45b57166","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.423824487Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions","method":"POST","req_id":"46d08a6a-bf37-48d6-8cdc-373f98f432cd","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":844,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.599236947Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"region":"de05e91b-b582-425e-840a-192a45b57166","component":"worker"}
{"msg":"region de05e91b-b582-425e-840a-192a45b57166 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.599340668Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.59953579Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.599928444Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.616480143Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.67194789Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.68392507Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.684692138Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.688789732Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.688847123Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.688900073Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.725846331Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:45.757827486Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:46.034418366Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions","method":"POST","req_id":"2373de82-efd4-4af8-8384-4a863afd0cb7","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.193194932Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions/de05e91b-b582-425e-840a-192a45b57166","method":"GET","req_id":"b379f347-72fe-4a72-9293-d58284eeeb5b","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"path is \"/data/regions/de05e91b-b582-425e-840a-192a45b57166\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.196018892Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"dataset is oxp_7d81c0ff-af0b-41a7-af78-d5743695642b/crucible/regions/de05e91b-b582-425e-840a-192a45b57166","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.205299812Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"region de05e91b-b582-425e-840a-192a45b57166 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.215274399Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.215494052Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.215578253Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions/de05e91b-b582-425e-840a-192a45b57166","method":"DELETE","req_id":"28cd83e0-b176-4b46-ac7f-5b0f745cfc6a","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19617,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-de05e91b-b582-425e-840a-192a45b57166 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.216703625Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.218409753Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions/de05e91b-b582-425e-840a-192a45b57166","method":"GET","req_id":"2cc253d3-2003-44ba-9adf-ba9617fa8dd9","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.240316719Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7d81c0ff-af0b-41a7-af78-d5743695642b/crucible/regions/de05e91b-b582-425e-840a-192a45b57166\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.250897903Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"region":"de05e91b-b582-425e-840a-192a45b57166","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.321454414Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions/de05e91b-b582-425e-840a-192a45b57166","method":"GET","req_id":"c241aadf-955c-44f1-b4ee-b445c71ae31f","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":721,"response_code":"200"}
{"msg":"region de05e91b-b582-425e-840a-192a45b57166 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.356546422Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-29T22:40:47.483751672Z","hostname":"oxz_crucible_bb307168-4f8f-4f5a-8252-1e92fed5977a","pid":21026,"uri":"/crucible/0/regions/de05e91b-b582-425e-840a-192a45b57166","method":"GET","req_id":"b7950713-cf09-4e00-a9f9-8196ceba760e","remote_addr":"[fd00:1122:3344:101::b]:61596","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":42,"response_code":"200"}