[ Nov 14 08:57:48 Enabled. ]
[ Nov 14 08:57:48 Rereading configuration. ]
[ Nov 14 08:57:53 Rereading configuration. ]
[ Nov 14 08:57:58 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_493c27e1-df39-477b-8315-ad9aa5767668/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_493c27e1-df39-477b-8315-ad9aa5767668/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:57:58.374793297Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:57:58.377785395Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:57:58.377801076Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:57:58.395094023Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:57:58.495906603Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.400531402Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54459"}
{"msg":"region 44c9bd0c-4f1e-4d61-9797-e8e9233f1d02 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.406916555Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.408558892Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions","method":"POST","req_id":"096727e8-a3d3-4fb4-8ea6-43913d991053","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3990,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.408577595Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_493c27e1-df39-477b-8315-ad9aa5767668/crucible/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.417320182Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_493c27e1-df39-477b-8315-ad9aa5767668/crucible/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.417354353Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"44c9bd0c-4f1e-4d61-9797-e8e9233f1d02\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.491029786Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"region":"44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"44c9bd0c-4f1e-4d61-9797-e8e9233f1d02\" \"--data\" \"/data/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.493054849Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"region":"44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.596414429Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions","method":"POST","req_id":"d111774f-223d-4a30-9d52-ee44b68c583b","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.619271557Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"region":"44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","component":"worker"}
{"msg":"region 44c9bd0c-4f1e-4d61-9797-e8e9233f1d02 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.619295667Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.619422719Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.619820605Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.624284477Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.634989309Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.637954827Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.638676858Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.63876816Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.63881682Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.638855811Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.643700049Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:23.647890756Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.226278898Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions","method":"POST","req_id":"f217a559-9047-4586-b8e1-db73b69517fc","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.540949463Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","method":"GET","req_id":"d6d9b40f-c05e-4992-8725-c59cd21e23ed","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"path is \"/data/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.543578635Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"dataset is oxp_493c27e1-df39-477b-8315-ad9aa5767668/crucible/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.554361793Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"region 44c9bd0c-4f1e-4d61-9797-e8e9233f1d02 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.564595223Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.564730345Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-44c9bd0c-4f1e-4d61-9797-e8e9233f1d02 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.565780912Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.566714217Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","method":"DELETE","req_id":"841a7749-fe51-4778-b037-28c73b5ac0d5","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23161,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.568968413Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","method":"GET","req_id":"a9019cb7-6452-4890-848d-535186d0ab28","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.56999202Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_493c27e1-df39-477b-8315-ad9aa5767668/crucible/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.579256728Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"region":"44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","component":"worker"}
{"msg":"region 44c9bd0c-4f1e-4d61-9797-e8e9233f1d02 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.64181632Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T08:58:24.673070536Z","hostname":"oxz_crucible_37b94fe2-7e49-4f8b-ab73-391d62dac7da","pid":23773,"uri":"/crucible/0/regions/44c9bd0c-4f1e-4d61-9797-e8e9233f1d02","method":"GET","req_id":"8e52cd3d-dbf5-41c3-b781-4343c078e951","remote_addr":"[fd00:1122:3344:101::c]:54459","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":88,"response_code":"200"}