[ Nov 28 01:52:58 Enabled. ]
[ Nov 28 01:52:58 Rereading configuration. ]
[ Nov 28 01:52:59 Rereading configuration. ]
[ Nov 28 01:53:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_da2163f3-63fc-4f7b-b4fe-cdac494080be/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_da2163f3-63fc-4f7b-b4fe-cdac494080be/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:04.407196835Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:04.410997161Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:04.411011141Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:04.428930212Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:04.607661937Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.087204313Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:48574"}
{"msg":"region 24ba0337-4e6f-4b21-b68c-0e343f68e684 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.095549429Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.095761672Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.100652687Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"uri":"/crucible/0/regions","method":"POST","req_id":"7c25ceba-5322-4050-82e8-fce9ab755b98","remote_addr":"[fd00:1122:3344:101::b]:48574","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":7125,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_da2163f3-63fc-4f7b-b4fe-cdac494080be/crucible/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.106805984Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_da2163f3-63fc-4f7b-b4fe-cdac494080be/crucible/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.106827144Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"24ba0337-4e6f-4b21-b68c-0e343f68e684\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.204884318Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"region":"24ba0337-4e6f-4b21-b68c-0e343f68e684","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"24ba0337-4e6f-4b21-b68c-0e343f68e684\" \"--data\" \"/data/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.206906113Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"region":"24ba0337-4e6f-4b21-b68c-0e343f68e684","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.324123049Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"region":"24ba0337-4e6f-4b21-b68c-0e343f68e684","component":"worker"}
{"msg":"region 24ba0337-4e6f-4b21-b68c-0e343f68e684 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.32417274Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.324253671Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.324643138Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.325383911Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"uri":"/crucible/0/regions","method":"POST","req_id":"8241c039-6108-4e4c-80fc-68cddaf8b47f","remote_addr":"[fd00:1122:3344:101::b]:48574","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.32883719Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.341366168Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.344948001Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.345701494Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.345765585Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.345810665Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.345853606Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.350560278Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:49.354521246Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.364358886Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"uri":"/crucible/0/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","method":"GET","req_id":"28db0e5e-9063-471e-b970-160c9155cdb4","remote_addr":"[fd00:1122:3344:101::b]:48574","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":99,"response_code":"200"}
{"msg":"path is \"/data/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.366284929Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"dataset is oxp_da2163f3-63fc-4f7b-b4fe-cdac494080be/crucible/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.377502484Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"region 24ba0337-4e6f-4b21-b68c-0e343f68e684 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.388110389Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.388270601Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.388343323Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"uri":"/crucible/0/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","method":"DELETE","req_id":"612d419a-91c5-4837-9e92-01d4ef826578","remote_addr":"[fd00:1122:3344:101::b]:48574","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22066,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-24ba0337-4e6f-4b21-b68c-0e343f68e684 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.391391336Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.391947085Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"uri":"/crucible/0/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","method":"GET","req_id":"b1f2b239-46fe-4ceb-a735-f628b1c9aaff","remote_addr":"[fd00:1122:3344:101::b]:48574","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.396050677Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_da2163f3-63fc-4f7b-b4fe-cdac494080be/crucible/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.408072835Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"region":"24ba0337-4e6f-4b21-b68c-0e343f68e684","component":"worker"}
{"msg":"region 24ba0337-4e6f-4b21-b68c-0e343f68e684 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.470736184Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:53:50.473836468Z","hostname":"oxz_crucible_d24eaff3-a59b-4fb7-89de-6ce285942026","pid":23295,"uri":"/crucible/0/regions/24ba0337-4e6f-4b21-b68c-0e343f68e684","method":"GET","req_id":"65d79224-b971-4caa-9981-74018daf035e","remote_addr":"[fd00:1122:3344:101::b]:48574","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}