[ Dec 21 06:13:38 Enabled. ]
[ Dec 21 06:13:39 Rereading configuration. ]
[ Dec 21 06:13:45 Rereading configuration. ]
[ Dec 21 06:13:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6ca8525c-d512-4829-b108-98cb3be6ac5a/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6ca8525c-d512-4829-b108-98cb3be6ac5a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:13:53.781997451Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:13:53.785631582Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:13:53.785641752Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:13:53.802724816Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:13:53.94278105Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.572644216Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:32773"}
{"msg":"region 34777e68-f62d-4e6a-a6f5-e869e5151be2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.595179721Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.59549356Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"uri":"/crucible/0/regions","method":"POST","req_id":"64735c3d-441e-479b-95ad-265ce7b6a536","remote_addr":"[fd00:1122:3344:101::c]:32773","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2399,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.59551243Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_6ca8525c-d512-4829-b108-98cb3be6ac5a/crucible/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.604539151Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6ca8525c-d512-4829-b108-98cb3be6ac5a/crucible/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.604597182Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"34777e68-f62d-4e6a-a6f5-e869e5151be2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.66869332Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"region":"34777e68-f62d-4e6a-a6f5-e869e5151be2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"34777e68-f62d-4e6a-a6f5-e869e5151be2\" \"--data\" \"/data/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.670716336Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"region":"34777e68-f62d-4e6a-a6f5-e869e5151be2","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.781401786Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"region":"34777e68-f62d-4e6a-a6f5-e869e5151be2","component":"worker"}
{"msg":"region 34777e68-f62d-4e6a-a6f5-e869e5151be2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.781479968Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.781645143Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.782033164Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.788363559Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.794300004Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"uri":"/crucible/0/regions","method":"POST","req_id":"c0fef736-6b72-4959-96a2-5a612e88743a","remote_addr":"[fd00:1122:3344:101::c]:32773","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.803414397Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.810851053Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.811581393Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.811642565Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.811693186Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.811723427Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.818009961Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:14.823026931Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.374431903Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"uri":"/crucible/0/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","method":"GET","req_id":"464fab1d-c4f5-4b2d-971e-f7fe2689c713","remote_addr":"[fd00:1122:3344:101::c]:32773","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.379171164Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"dataset is oxp_6ca8525c-d512-4829-b108-98cb3be6ac5a/crucible/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.390495018Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"region 34777e68-f62d-4e6a-a6f5-e869e5151be2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.400979469Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.401220126Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.401289018Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"uri":"/crucible/0/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","method":"DELETE","req_id":"92b5bd20-2cd8-4cb7-b5af-72fda3e93b36","remote_addr":"[fd00:1122:3344:101::c]:32773","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22147,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-34777e68-f62d-4e6a-a6f5-e869e5151be2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.402226164Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.404585509Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"uri":"/crucible/0/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","method":"GET","req_id":"7c423ef9-321e-4afb-8dd9-43076cfa5bfa","remote_addr":"[fd00:1122:3344:101::c]:32773","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":39,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.407108299Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6ca8525c-d512-4829-b108-98cb3be6ac5a/crucible/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.418371482Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"region":"34777e68-f62d-4e6a-a6f5-e869e5151be2","component":"worker"}
{"msg":"region 34777e68-f62d-4e6a-a6f5-e869e5151be2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.493896076Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T06:14:15.535569082Z","hostname":"oxz_crucible_bb9218ea-cb92-478f-b10b-6d0ba7a53c59","pid":23060,"uri":"/crucible/0/regions/34777e68-f62d-4e6a-a6f5-e869e5151be2","method":"GET","req_id":"5101b019-b798-4d4d-bdc5-16993a9b9c9f","remote_addr":"[fd00:1122:3344:101::c]:32773","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}