For the record when built with -fsanitize=address the report is:

=================================================================
==909==ERROR: AddressSanitizer: attempting free on address which was not malloc()-ed: 0xb6f14f30 in thread T0
    #0 0xb6a12f84 in operator delete(void*) (/lib/libasan.so.5+0x113f84)
    #1 0x265f8 in __gnu_cxx::new_allocator<char>::deallocate(char*, unsigned int) /usr/include/c++/9/ext/new_allocator.h:128
    #2 0x265f8 in std::allocator_traits<std::allocator<char> >::deallocate(std::allocator<char>&, char*, unsigned int) /usr/include/c++/9/bits/alloc_traits.h:470
    #3 0x265f8 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_destroy(unsigned int) /usr/include/c++/9/bits/basic_string.h:237
    #4 0x265f8 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose() /usr/include/c++/9/bits/basic_string.h:232
    #5 0x265f8 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() /usr/include/c++/9/bits/basic_string.h:658
    #6 0x265f8 in Catch::AssertionResultData::~AssertionResultData() /builddir/ost2/catch.hpp:956
    #7 0x265f8 in Catch::ResultBuilder::~ResultBuilder() /builddir/ost2/catch.hpp:9188
    #8 0x21674 in ____C_A_T_C_H____T_E_S_T____0 /builddir/ost2/test_setup.cpp:11
    #9 0x6d66c in Catch::FreeFunctionTestCase::invoke() const /builddir/ost2/catch.hpp:7415
    #10 0x6d66c in Catch::TestCase::invoke() const /builddir/ost2/catch.hpp:8413
    #11 0x6d66c in Catch::RunContext::invokeActiveTestCase() /builddir/ost2/catch.hpp:6941
    #12 0x6d66c in Catch::RunContext::runCurrentTest(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) /builddir/ost2/catch.hpp:6914
    #13 0x6d66c in Catch::RunContext::runTest(Catch::TestCase const&) /builddir/ost2/catch.hpp:6706
    #14 0x6d66c in Catch::runTests(Catch::Ptr<Catch::Config> const&) /builddir/ost2/catch.hpp:7086
    #15 0x1b7e0 in Catch::Session::runInternal() /builddir/ost2/catch.hpp:7258
    #16 0x1b7e0 in Catch::Session::run() /builddir/ost2/catch.hpp:7217
    #17 0x1b7e0 in Catch::Session::run(int, char const* const*) /builddir/ost2/catch.hpp:7182
    #18 0x1b7e0 in main /builddir/ost2/catch.hpp:11469
    #19 0xb6500878 in __libc_start_main /usr/src/debug/glibc-2.28.9000-564-g008b598e2a/csu/libc-start.c:308

Address 0xb6f14f30 is a wild pointer.
SUMMARY: AddressSanitizer: bad-free (/lib/libasan.so.5+0x113f84) in operator delete(void*)
==909==ABORTING

Which appears to show a bogus free from a std::string destructor without any previously reported issues.

I haven't been able to get valgrind to work in a mock chroot on armv7hl to see what that says about the non-sanitized version.

Bisected to http://gcc.gnu.org/r266385 change, unfortunately the amount of changes due to this RA change is huge:
diff -up rh1670069-2.s.g26638{4,5} | diffstat
 rh1670069-2.s.g266385 |29389 +++++++++++++++++++++++++-------------------------
 1 file changed, 14888 insertions(+), 14501 deletions(-)
(that is with -g0).

So, I've used following patch to bisect within the large testcase:
--- gcc/ira-costs.c.jj	2019-01-16 09:35:09.132247497 +0100
+++ gcc/ira-costs.c	2019-01-29 11:42:56.643422968 +0100
@@ -1277,16 +1277,22 @@ record_address_regs (machine_mode mode,
 
 

 
+static int xcntx = -1;
+static int xlimitx = -1;
+
 /* Calculate the costs of insn operands.  */
 static void
 record_operand_costs (rtx_insn *insn, enum reg_class *pref)
 {
   const char *constraints[MAX_RECOG_OPERANDS];
   machine_mode modes[MAX_RECOG_OPERANDS];
+  rtx ops[MAX_RECOG_OPERANDS];
   rtx set;
   int i;
+  bool old = xcntx < xlimitx;
 
-  if ((set = single_set (insn)) != NULL_RTX
+  if (!old
+      && (set = single_set (insn)) != NULL_RTX
       /* In rare cases the single set insn might have less 2 operands
 	 as the source can be a fixed special reg.  */
       && recog_data.n_operands > 1
@@ -1404,6 +1410,7 @@ record_operand_costs (rtx_insn *insn, en
     {
       memcpy (op_costs[i], init_cost, struct_costs_size);
 
+      ops[i] = recog_data.operand[i];
       if (GET_CODE (recog_data.operand[i]) == SUBREG)
 	recog_data.operand[i] = SUBREG_REG (recog_data.operand[i]);
 
@@ -1443,6 +1450,56 @@ record_operand_costs (rtx_insn *insn, en
   record_reg_classes (recog_data.n_alternatives, recog_data.n_operands,
 		      recog_data.operand, modes,
 		      constraints, insn, pref);
+
+  if (old
+      && (set = single_set (insn)) != NULL_RTX
+      /* In rare cases the single set insn might have less 2 operands
+	 as the source can be a fixed special reg.  */
+      && recog_data.n_operands > 1
+      && ops[0] == SET_DEST (set) && ops[1] == SET_SRC (set))
+    {
+      int regno, other_regno;
+      rtx dest = SET_DEST (set);
+      rtx src = SET_SRC (set);
+
+      if (GET_CODE (dest) == SUBREG
+	  && known_eq (GET_MODE_SIZE (GET_MODE (dest)),
+		       GET_MODE_SIZE (GET_MODE (SUBREG_REG (dest)))))
+	dest = SUBREG_REG (dest);
+      if (GET_CODE (src) == SUBREG
+	  && known_eq (GET_MODE_SIZE (GET_MODE (src)),
+		       GET_MODE_SIZE (GET_MODE (SUBREG_REG (src)))))
+	src = SUBREG_REG (src);
+      if (REG_P (src) && REG_P (dest)
+	  && find_regno_note (insn, REG_DEAD, REGNO (src))
+	  && (((regno = REGNO (src)) >= FIRST_PSEUDO_REGISTER
+	       && (other_regno = REGNO (dest)) < FIRST_PSEUDO_REGISTER)
+	      || ((regno = REGNO (dest)) >= FIRST_PSEUDO_REGISTER
+		  && (other_regno = REGNO (src)) < FIRST_PSEUDO_REGISTER)))
+	{
+	  machine_mode mode = GET_MODE (src);
+	  cost_classes_t cost_classes_ptr = regno_cost_classes[regno];
+	  enum reg_class *cost_classes = cost_classes_ptr->classes;
+	  reg_class_t rclass;
+	  int k;
+
+	  i = regno == (int) REGNO (src) ? 1 : 0;
+	  for (k = cost_classes_ptr->num - 1; k >= 0; k--)
+	    {
+	      rclass = cost_classes[k];
+	      if (TEST_HARD_REG_BIT (reg_class_contents[rclass], other_regno)
+		  && (reg_class_size[(int) rclass]
+		      == ira_reg_class_max_nregs [(int) rclass][(int) mode]))
+		{
+		  if (reg_class_size[rclass] == 1)
+		    op_costs[i]->cost[k] = -frequency;
+		  else if (in_hard_reg_set_p (reg_class_contents[rclass],
+					      mode, other_regno))
+		    op_costs[i]->cost[k] = -frequency;
+		}
+	    }
+	}
+    }
 }
 
 

@@ -2289,6 +2346,8 @@ finish_costs (void)
 void
 ira_costs (void)
 {
+if (xlimitx == -1 && getenv ("IRA_COSTS")) xlimitx = atoi (getenv ("IRA_COSTS"));
+xcntx++;
   allocno_p = true;
   cost_elements_num = ira_allocnos_num;
   init_costs ();

which conditionally reverts the r266385 changes for selected functions.  Bisection found that
IRA_COSTS=848 ./cc1plus -quiet -nostdinc -O2 -fstack-protector-strong -fPIE -march=armv7-a -mfpu=vfpv3-d16 -mtune=generic-armv7-a -mabi=aapcs-linux -mfloat-abi=hard rh1670069-2.ii -o rh1670069-2.s
still fails while
IRA_COSTS=849 ./cc1plus -quiet -nostdinc -O2 -fstack-protector-strong -fPIE -march=armv7-a -mfpu=vfpv3-d16 -mtune=generic-armv7-a -mabi=aapcs-linux -mfloat-abi=hard rh1670069-2.ii -o rh1670069-2.s
works, the difference is in _ZN10CommandCat5setupERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS6_EE, i.e.
CommandCat::setup(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&)
function:
--- rh1670069-2.s.bad	2019-01-29 12:01:58.152351265 +0100
+++ rh1670069-2.s.good	2019-01-29 12:03:34.971743525 +0100
@@ -104083,65 +104083,62 @@ _ZNSt8_Rb_treeINSt7__cxx1112basic_string
 _ZN10CommandCat5setupERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS6_EE:
 	.fnstart
 .LFB6911:
-	@ args = 0, pretend = 0, frame = 680
+	@ args = 0, pretend = 0, frame = 704
 	@ frame_needed = 0, uses_anonymous_args = 0
+	mov	r3, r0
 	push	{r4, r5, r6, r7, r8, r9, r10, fp, lr}
 	.save {r4, r5, r6, r7, r8, r9, r10, fp, lr}
-	vpush.64	{d8, d9, d10}
-	.vsave {d8, d9, d10}
-	mov	r3, r0
-	.pad #684
-	sub	sp, sp, #684
 	ldr	r2, .L16750
-	str	r3, [sp, #8]
+	.pad #708
+	sub	sp, sp, #708
+	str	r3, [sp, #16]
 	ldr	r3, .L16750+4
 .LPIC1872:
 	add	r2, pc, r2
 	mov	r4, r1
-	ldr	r6, .L16750+8
-	ldr	r1, .L16750+12
+	ldr	r1, .L16750+8
 	ldr	r3, [r2, r3]
-	add	r5, sp, #600
-.LPIC1865:
-	add	r6, pc, r6
+	add	r5, sp, #624
 .LPIC1864:
 	add	r1, pc, r1
 	mov	r0, r5
+	ldr	r10, .L16750+12
 	ldr	r3, [r3]
-	str	r3, [sp, #676]
+	str	r3, [sp, #700]
 	mov	r3,#0
-	str	r6, [sp, #12]
 .LEHB2076:
 	bl	_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEC2EPKcRKS3_.isra.0(PLT)
 .LEHE2076:
 	ldr	r3, .L16750+16
-	add	r8, sp, #220
-	ldr	r3, [r6, r3]
-	mov	r1, r5
+.LPIC1865:
+	add	r10, pc, r10
+	ldr	r3, [r10, r3]
+	add	fp, sp, #244
 	ldr	r7, [r3]
-	mov	r0, r8
-	lsr	fp, r7, #1
-	mov	r3, fp
+	mov	r1, r5
+	lsr	r8, r7, #1
+	mov	r0, fp
+	mov	r3, r8
 	mov	r2, r7
 .LEHB2077:
 	bl	_ZN5boost15program_options19options_descriptionC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjj(PLT)
 .LEHE2077:
-	ldr	r0, [sp, #600]
-	add	r3, sp, #608
+	ldr	r0, [sp, #624]
+	add	r3, sp, #632
 	cmp	r0, r3
 	beq	.L16671
 	bl	_ZdlPv(PLT)
 .L16671:
-	mov	r0, r8
+	mov	r0, fp
 .LEHB2078:
 	bl	_ZN5boost15program_options19options_description11add_optionsEv(PLT)
 	mov	r3, r0
 	mov	r0, #0
-	str	r3, [sp, #40]
+	str	r3, [sp, #64]
 	bl	_ZN5boost15program_options5valueISt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS8_EEEEPNS0_11typed_valueIT_cEEPSC_(PLT)
 	ldr	r3, .L16750+20
 	ldr	r1, .L16750+24
-	add	r6, sp, #40
+	add	r6, sp, #64
 	mov	r2, r0
 .LPIC1866:
 	add	r3, pc, r3
@@ -104149,24 +104146,25 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 .LPIC1867:
 	add	r1, pc, r1
 	bl	_ZN5boost15program_options29options_description_easy_initclEPKcPKNS0_14value_semanticES3_(PLT)
-	add	r3, sp, #296
-	vmov	s20, r3	@ int
-	ldr	r9, [sp, #8]
-	vmov	r0, s20	@ int
+	ldr	r9, [sp, #16]
+	add	r3, sp, #320
+	mov	r0, r3
 	add	r1, r9, #8
+	str	r3, [sp, #8]
 	bl	_ZN24with_multiple_osm_inputs27add_multiple_inputs_optionsEv(PLT)
 .LEHE2078:
-	add	r3, sp, #372
-	vmov	s21, r3	@ int
+	add	r3, sp, #396
+	mov	r0, r3
 	add	r1, r9, #20
-	vmov	r0, s21	@ int
+	str	r3, [sp, #12]
 .LEHB2079:
 	bl	_ZN15with_osm_output18add_output_optionsEv(PLT)
 .LEHE2079:
-	add	r9, sp, #448
+	add	r9, sp, #472
 	mov	r0, r9
-	mov	r2, fp
+	mov	r2, r8
 	mov	r1, r7
+	str	r9, [sp, #20]
 .LEHB2080:
 	bl	_ZN5boost15program_options19options_descriptionC1Ejj(PLT)
 .LEHE2080:
@@ -104175,7 +104173,7 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	bl	_ZN5boost15program_options19options_description11add_optionsEv(PLT)
 	mov	r3, r0
 	mov	r0, #0
-	str	r3, [sp, #40]
+	str	r3, [sp, #64]
 	bl	_ZN5boost15program_options5valueISt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS8_EEEEPNS0_11typed_valueIT_cEEPSC_(PLT)
 	ldr	r3, .L16750+28
 	ldr	r1, .L16750+32
@@ -104186,38 +104184,41 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 .LPIC1869:
 	add	r1, pc, r1
 	bl	_ZN5boost15program_options29options_description_easy_initclEPKcPKNS0_14value_semanticES3_(PLT)
-	add	r10, sp, #524
-	mov	r0, r10
-	mov	r2, fp
+	add	r3, sp, #548
+	mov	r0, r3
+	mov	r2, r8
 	mov	r1, r7
+	str	r3, [sp, #4]
 	bl	_ZN5boost15program_options19options_descriptionC1Ejj(PLT)
 .LEHE2081:
-	mov	r1, r8
-	mov	r0, r10
+	mov	r1, fp
+	ldr	r0, [sp, #4]
 .LEHB2082:
 	bl	_ZN5boost15program_options19options_description3addERKS1_(PLT)
-	vmov	r1, s20	@ int
+	ldr	r1, [sp, #8]
 	bl	_ZN5boost15program_options19options_description3addERKS1_(PLT)
-	vmov	r1, s21	@ int
+	ldr	r1, [sp, #12]
 	bl	_ZN5boost15program_options19options_description3addERKS1_(PLT)
-	mov	r2, fp
+	mov	r2, r8
 	mov	r1, r7
 	mov	r0, r5
 	bl	_ZN5boost15program_options19options_descriptionC1Ejj(PLT)
 .LEHE2082:
-	mov	r1, r10
 	mov	r0, r5
+	ldr	r1, [sp, #4]
 .LEHB2083:
 	bl	_ZN5boost15program_options19options_description3addERKS1_(PLT)
 	mov	r1, r9
 	bl	_ZN5boost15program_options19options_description3addERKS1_(PLT)
-	add	fp, sp, #120
-	mov	r0, fp
+	add	r3, sp, #144
+	mov	r0, r3
+	mov	r8, r3
+	str	r3, [sp, #24]
 	bl	_ZN5boost15program_options30positional_options_descriptionC1Ev(PLT)
 .LEHE2083:
 	ldr	r1, .L16750+36
 	mvn	r2, #0
-	mov	r0, fp
+	mov	r0, r8
 .LPIC1870:
 	add	r1, pc, r1
 .LEHB2084:
@@ -104225,7 +104226,7 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	mov	r0, r6
 	bl	_ZN5boost15program_options13variables_mapC1Ev(PLT)
 .LEHE2084:
-	add	r7, sp, #156
+	add	r7, sp, #180
 	mov	r1, r4
 	mov	r0, r7
 .LEHB2085:
@@ -104235,11 +104236,11 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	mov	r0, r7
 .LEHB2086:
 	bl	_ZN5boost15program_options6detail7cmdline23set_options_descriptionERKNS0_19options_descriptionE(PLT)
-	mov	r1, fp
+	mov	r1, r8
 	mov	r0, r7
-	str	r5, [sp, #216]
+	str	r5, [sp, #240]
 	bl	_ZN5boost15program_options6detail7cmdline22set_positional_optionsERKNS0_30positional_options_descriptionE(PLT)
-	add	r4, sp, #20
+	add	r4, sp, #44
 	mov	r0, r4
 	mov	r1, r7
 	bl	_ZN5boost15program_options25basic_command_line_parserIcE3runEv(PLT)
@@ -104250,60 +104251,56 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 .LEHB2087:
 	bl	_ZN5boost15program_options5storeERKNS0_20basic_parsed_optionsIcEERNS0_13variables_mapEb(PLT)
 .LEHE2087:
-	ldr	r2, [sp, #24]
-	ldr	r3, [sp, #20]
-	str	r2, [sp, #4]
-	cmp	r3, r2
+	ldrd	r8, [sp, #44]
+	cmp	r8, r9
 	beq	.L16672
-	vmov	s16, r5	@ int
-	vmov	s17, r8	@ int
-	mov	r5, r3
-	vmov	s18, r7	@ int
-	vmov	s19, r6	@ int
-	add	r4, r3, #8
+	add	r4, r8, #8
+	str	r5, [sp, #28]
+	str	r7, [sp, #32]
+	str	r6, [sp, #36]
 .L16682:
-	ldr	r8, [r4, #32]
-	ldr	r6, [r4, #36]
-	cmp	r8, r6
+	ldr	r7, [r4, #32]
+	ldr	r5, [r4, #36]
+	cmp	r7, r5
 	beq	.L16673
-	add	r7, r8, #8
+	add	r6, r7, #8
 .L16675:
-	ldr	r0, [r7, #-8]
-	cmp	r7, r0
+	ldr	r0, [r6, #-8]
+	cmp	r6, r0
 	beq	.L16674
 	bl	_ZdlPv(PLT)
 .L16674:
-	add	r8, r8, #24
-	cmp	r6, r8
 	add	r7, r7, #24
+	cmp	r5, r7
+	add	r6, r6, #24
 	bne	.L16675
-	ldr	r6, [r4, #32]
+	ldr	r5, [r4, #32]
 .L16673:
-	cmp	r6, #0
+	cmp	r5, #0
 	beq	.L16676
-	mov	r0, r6
+	mov	r0, r5
 	bl	_ZdlPv(PLT)
 .L16676:
-	ldr	r8, [r4, #20]
-	ldr	r6, [r4, #24]
-	cmp	r8, r6
+	ldr	r7, [r4, #20]
+	ldr	r5, [r4, #24]
+	cmp	r7, r5
 	beq	.L16677
-	add	r7, r8, #8
+	add	r6, r7, #8
 .L16679:
-	ldr	r0, [r7, #-8]
-	cmp	r7, r0
+	ldr	r0, [r6, #-8]
+	cmp	r6, r0
 	beq	.L16678
 	bl	_ZdlPv(PLT)
 .L16678:
-	add	r8, r8, #24
-	cmp	r6, r8
 	add	r7, r7, #24
+	cmp	r5, r7
+	add	r6, r6, #24
 	bne	.L16679
-	ldr	r6, [r4, #20]
+	ldr	r5, [r4, #20]
 .L16677:
-	cmp	r6, #0
+	cmp	r5, #0
 	beq	.L16680
-	mov	r0, r6
+	mov	r0, r5
 	bl	_ZdlPv(PLT)
 .L16680:
 	ldr	r0, [r4, #-8]
@@ -104311,25 +104308,21 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	beq	.L16681
 	bl	_ZdlPv(PLT)
 .L16681:
-	ldr	r3, [sp, #4]
-	add	r5, r5, #56
-	cmp	r3, r5
+	add	r8, r8, #56
+	cmp	r9, r8
 	add	r4, r4, #56
 	bne	.L16682
-	vmov	r8, s17	@ int
-	vmov	r5, s16	@ int
-	vmov	r7, s18	@ int
-	vmov	r6, s19	@ int
-	ldr	r3, [sp, #20]
-	str	r3, [sp, #4]
+	ldr	r5, [sp, #28]
+	ldr	r7, [sp, #32]
+	ldr	r6, [sp, #36]
+	ldr	r9, [sp, #44]
 .L16672:
-	ldr	r3, [sp, #4]
-	cmp	r3, #0
+	cmp	r9, #0
 	beq	.L16683
-	mov	r0, r3
+	mov	r0, r9
 	bl	_ZdlPv(PLT)
 .L16683:
-	ldr	r3, [sp, #200]
+	ldr	r3, [sp, #224]
 	cmp	r3, #0
 	beq	.L16684
 	tst	r3, #1
@@ -104338,12 +104331,12 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	ldr	r3, [r3]
 	cmp	r3, #0
 	beq	.L16684
-	add	r1, sp, #204
+	add	r1, sp, #228
 	mov	r0, r1
 	mov	r2, #2
 	blx	r3
 .L16684:
-	ldr	r3, [sp, #184]
+	ldr	r3, [sp, #208]
 	cmp	r3, #0
 	beq	.L16685
 	tst	r3, #1
@@ -104355,42 +104348,41 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 .LEHB2088:
 	bl	_ZN5boost15program_options6notifyERNS0_13variables_mapE(PLT)
 	mov	r1, r6
-	ldr	r0, [sp, #8]
+	ldr	r0, [sp, #16]
 	bl	_ZN7Command22setup_object_type_nwrcERKN5boost15program_options13variables_mapE(PLT)
 .LEHE2088:
-	ldr	r2, [sp, #12]
 	ldr	r3, .L16750+40
-	ldr	r1, [sp, #104]
-	ldr	r3, [r2, r3]
-	add	r0, sp, #96
+	ldr	r1, [sp, #128]
+	ldr	r3, [r10, r3]
+	add	r0, sp, #120
 	add	r3, r3, #8
-	str	r3, [sp, #40]
+	str	r3, [sp, #64]
 	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE8_M_eraseEPSt13_Rb_tree_nodeIS8_E(PLT)
-	ldr	r1, [sp, #80]
-	add	r0, sp, #72
+	ldr	r1, [sp, #104]
+	add	r0, sp, #96
 	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St9_IdentityIS5_ESt4lessIS5_ESaIS5_EE8_M_eraseEPSt13_Rb_tree_nodeIS5_E(PLT)
-	add	r0, sp, #48
-	ldr	r1, [sp, #56]
+	add	r0, sp, #72
+	ldr	r1, [sp, #80]
 	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_N5boost15program_options14variable_valueEESt10_Select1stISB_ESt4lessIS5_ESaISB_EE8_M_eraseEPSt13_Rb_tree_nodeISB_E(PLT)
-	ldr	r0, [sp, #132]
-	add	r3, sp, #140
+	ldr	r0, [sp, #156]
+	add	r3, sp, #164
 	cmp	r0, r3
 	beq	.L16686
 	bl	_ZdlPv(PLT)
 .L16686:
-	mov	r0, fp
+	ldr	r0, [sp, #24]
 	bl	_ZNSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS5_EED1Ev(PLT)
 	mov	r0, r5
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
-	mov	r0, r10
+	ldr	r0, [sp, #4]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
-	mov	r0, r9
+	ldr	r0, [sp, #20]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
-	vmov	r0, s21	@ int
+	ldr	r0, [sp, #12]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
-	vmov	r0, s20	@ int
+	ldr	r0, [sp, #8]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
-	mov	r0, r8
+	mov	r0, fp
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 	ldr	r2, .L16750+44
 	ldr	r3, .L16750+4
@@ -104399,19 +104391,18 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	ldr	r3, [r2, r3]
 	mov	r0, #1
 	ldr	r2, [r3]
-	ldr	r3, [sp, #676]
+	ldr	r3, [sp, #700]
 	eors	r2, r3, r2
 	bne	.L16747
-	add	sp, sp, #684
+	add	sp, sp, #708
 	@ sp needed
-	vldm	sp!, {d8-d10}
 	pop	{r4, r5, r6, r7, r8, r9, r10, fp, pc}
 .L16746:
 	bic	r3, r3, #1
 	ldr	r3, [r3]
 	cmp	r3, #0
 	beq	.L16685
-	add	r1, sp, #188
+	add	r1, sp, #212
 	mov	r0, r1
 	mov	r2, #2
 	blx	r3
@@ -104419,18 +104410,18 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 .L16747:
 	bl	__stack_chk_fail(PLT)
 .L16705:
-	ldr	r0, [sp, #600]
-	add	r3, sp, #608
+	ldr	r0, [sp, #624]
+	add	r3, sp, #632
 	cmp	r0, r3
 	beq	.L16745
 	bl	_ZdlPv(PLT)
 	b	.L16745
 .L16707:
 .L16702:
-	vmov	r0, s20	@ int
+	ldr	r0, [sp, #8]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 .L16703:
-	mov	r0, r8
+	mov	r0, fp
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 .L16745:
 .LEHB2089:
@@ -104445,46 +104436,45 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 	mov	r0, r7
 	bl	_ZN5boost15program_options6detail7cmdlineD2Ev(PLT)
 .L16691:
-	ldr	r2, [sp, #12]
 	ldr	r3, .L16750+40
-	ldr	r1, [sp, #104]
-	ldr	r3, [r2, r3]
-	add	r0, sp, #96
+	ldr	r1, [sp, #128]
+	ldr	r3, [r10, r3]
+	add	r0, sp, #120
 	add	r3, r3, #8
-	str	r3, [sp, #40]
+	str	r3, [sp, #64]
 	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE8_M_eraseEPSt13_Rb_tree_nodeIS8_E(PLT)
+	ldr	r1, [sp, #104]
+	add	r0, sp, #96
+	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St9_IdentityIS5_ESt4lessIS5_ESaIS5_EE8_M_eraseEPSt13_Rb_tree_nodeIS5_E(PLT)
 	ldr	r1, [sp, #80]
 	add	r0, sp, #72
-	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St9_IdentityIS5_ESt4lessIS5_ESaIS5_EE8_M_eraseEPSt13_Rb_tree_nodeIS5_E(PLT)
-	ldr	r1, [sp, #56]
-	add	r0, sp, #48
 	bl	_ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_N5boost15program_options14variable_valueEESt10_Select1stISB_ESt4lessIS5_ESaISB_EE8_M_eraseEPSt13_Rb_tree_nodeISB_E(PLT)
 .L16692:
-	ldr	r0, [sp, #132]
-	add	r3, sp, #140
+	ldr	r0, [sp, #156]
+	add	r3, sp, #164
 	cmp	r0, r3
 	beq	.L16693
 	bl	_ZdlPv(PLT)
 .L16693:
-	ldr	r4, [sp, #120]
-	ldr	r6, [sp, #124]
+	ldr	r4, [sp, #144]
+	ldr	r6, [sp, #148]
 .L16696:
 	cmp	r6, r4
 	bne	.L16748
-	ldr	r0, [sp, #120]
+	ldr	r0, [sp, #144]
 	cmp	r0, #0
 	bne	.L16749
 .L16698:
 	mov	r0, r5
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 .L16699:
-	mov	r0, r10
+	ldr	r0, [sp, #4]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 .L16700:
-	mov	r0, r9
+	ldr	r0, [sp, #20]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 .L16701:
-	vmov	r0, s21	@ int
+	ldr	r0, [sp, #12]
 	bl	_ZN5boost15program_options19options_descriptionD1Ev(PLT)
 	b	.L16702
 .L16714:
@@ -104518,8 +104508,8 @@ _ZN10CommandCat5setupERKSt6vectorINSt7__
 .L16750:
 	.word	_GLOBAL_OFFSET_TABLE_-(.LPIC1872+8)
 	.word	__stack_chk_guard(GOT)
-	.word	_GLOBAL_OFFSET_TABLE_-(.LPIC1865+8)
 	.word	.LC424-(.LPIC1864+8)
+	.word	_GLOBAL_OFFSET_TABLE_-(.LPIC1865+8)
 	.word	_ZN5boost15program_options19options_description21m_default_line_lengthE(GOT)
 	.word	.LC425-(.LPIC1866+8)
 	.word	.LC426-(.LPIC1867+8)
Guess I'll need to find out now how many times this is called and do a side-by-side debugging section to find out what is different.

I believe that's only going to be called twice, which is something at least...

Created attachment 1524596 [details]
rh1670069.tar.xz

Tarball with the adjusted *.ii files and corresponding *.s (good/bad for the second one).
The failure seems to happen after the first setup method returned (and goes away if rh1670069-1.ii is rebuilt with -O0 g++ 8.2), so either there is some use after scope somewhere, or stack corruption.

One additional data point re the stack is that it seemed to go away without -fstack-protector-strong but that might just be coincidence in that it changed the memory layout.

Created attachment 1524602 [details]
rh1670069.tar.xz

Oops, forgot to build -1.ii with -g, this updated tarball has that, so one can step through the caller too.
In any case, would appreciate help with finding out what exactly gets clobbered, so far I do not see that :(.

Tried to use mtrace, but didn't find any frees of something not allocated by malloc.
The crash is on:
#0  0x0001d028 in std::char_traits<char>::copy (__n=130500, __s2=0xbeffe7f8 "L_\a", __s1=0x82878 "") at rh1670069-2.ii:10347
#1  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_S_copy (__n=130500, __s=0xbeffe7f8 "L_\a", __d=0x82878 "")
    at rh1670069-2.ii:16299
#2  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_S_copy (__n=130500, __s=0xbeffe7f8 "L_\a", __d=0x82878 "")
    at rh1670069-2.ii:16294
#3  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_S_copy_chars (__k2=<optimized out>, __k1=0xbeffe7f8 "L_\a", 
    __p=0x82878 "") at rh1670069-2.ii:16341
#4  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*> (this=0xbeffe64c, this@entry=0xbeffe644, 
    __beg=0xbeffe7f8 "L_\a", __end=<optimized out>) at rh1670069-2.ii:19293
#5  0x0001f850 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct_aux<char*> (
    __end=<optimized out>, __beg=<optimized out>, this=0xbeffe644) at rh1670069-2.ii:121486
#6  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*> (__end=<optimized out>, 
    __beg=<optimized out>, this=0xbeffe644) at rh1670069-2.ii:16229
#7  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string (__str=..., this=0xbeffe644)
    at rh1670069-2.ii:16399
#8  Catch::AssertionResultData::AssertionResultData (this=0xbeffe640) at rh1670069-2.ii:45964
#9  Catch::ResultBuilder::build (this=this@entry=0xbeffe688, expr=...) at rh1670069-2.ii:55950
#10 0x0001fdc4 in Catch::ResultBuilder::build (this=0xbeffe688) at rh1670069-2.ii:121474
#11 Catch::ResultBuilder::captureExpression (this=this@entry=0xbeffe688) at rh1670069-2.ii:55912
#12 0x0001fe44 in Catch::ResultBuilder::captureResult (this=this@entry=0xbeffe688, resultType=resultType@entry=Catch::ResultWas::Ok)
    at rh1670069-2.ii:121421
#13 0x0001c1e8 in ::____C_A_T_C_H____T_E_S_T__ () at rh1670069-1.ii:80981
where 0xbeffe7f8 is on the stack and end of stack is 0xbf000000, so there are 6152 bytes on the stack, so trying to memcpy 130500 bytes from it segfaults.

I'm trying to figure out which string that is constructing but those line numbers don't make any sense :-(

You need the *.ii files from the tarball, I've intentionally removed all line notes from them, so that I don't need the original sources which I don't have.

Yes I'm using those, but rh1670069-2.ii:45964 is nowhere near Catch::AssertionResultData::AssertionResultData for example... That function is around line 111502 in reality.

Does the test throw an exception and catches it, by chance?

The Arm unwinder has a bug where it can clobber d8, which is used by the bad code in comment 3.

Yes the first test throws an exception and catches it!

Indeed I think that crash is exactly in the path where it is catching the exception - the involved string appears to be reconstructedExpression but at this point I doubt that matters.

The 0xbeffe7f8 address is right after the __catchResult variable (i.e. &__catchResult + 1), and you are right, that pointer is referenced in
reconstructedExpression = {static npos = 4294967295, 
      _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 
        _M_p = 0xbeffe7f8 "L_\a"}, _M_string_length = 130500, {
        _M_local_buf = "\\\347\377\276\204\347\377\276\370\347\377\276\330\325", <incomplete sequence \375\266>, 
        _M_allocated_capacity = 3204441948}}

Right, even though it looks like default construction I think it must be copy construction of an AssertionResultData object caused by this line in ResultBuilder::build:

    AssertionResultData data = m_data;

which is in turn copy constructing the reconstructedExpression member.

The default construction would default initialise the string so there wouldn't be any copying to do.

(In reply to Tom Hughes from comment #13)
> Yes the first test throws an exception and catches it!

(In reply to Tom Hughes from comment #14)
> Indeed I think that crash is exactly in the path where it is catching the
> exception - the involved string appears to be reconstructedExpression but at
> this point I doubt that matters.

Okay, then this might be the same bug we saw in glibc earlier: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89093

Can you run the test binaries against libstdc++ from Fedora 29?  Unfortunately, there does not seem to be a really good way to verify that this is the same bug.

You could try setting breakpoints on the loads from s16/s17 (which correspond to d8, “vmov <something>, s16” and “vmov <something>, s17”).  And then compare with the binary running on Fedora 29 and see if the register has different content there.

Unfortunately, with the glibc test case, I ran into GDB bugs setting those breakpoints, so I don't know if this feasible in practice.  There, d8 (s16/s17) was set to zero, but that could have been due to the particular nature of the unwind data in this case.

Sadly the rawhide compiled binaries won't run against the F29 library:

./rh1670069: /builddir/rh1670069/libstdc++.so.6: version `GLIBCXX_3.4.26' not found (required by ./rh1670069)

Will see if I can do the other thing...

...and of course those gcc 9 generated .i files won't compile with gcc 8 on F29 :-(

(In reply to Tom Hughes from comment #18)
> Sadly the rawhide compiled binaries won't run against the F29 library:
> 
> ./rh1670069: /builddir/rh1670069/libstdc++.so.6: version `GLIBCXX_3.4.26'
> not found (required by ./rh1670069)

I will try to produce a patched libstdc++.so.6 which works around this bug.

Actually, I've just replaced eh_personality.o in the libstdc++.so.6.0.26 I was using with one built with pre-r266385 where it doesn't save/restore d8 and the unit_tests.bad binary that previously failed now passes.
So, this might be indeed http://gcc.gnu.org/PR89093 in action.

Created attachment 1524631 [details]
Manually patched libstdc++.so.6.0.26

I patched the attached libstdc++ like this:

@@ -1,6 +1,6 @@
 0007b5b0 <__gxx_personality_v0@@CXXABI_1.3>:
    7b5b0:	e92d4ff0 	push	{r4, r5, r6, r7, r8, r9, sl, fp, lr}
-   7b5b4:	ed2d8b02 	vpush	{d8}
+   7b5b4:	ed2dfb02 	vpush	{d15}
    7b5b8:	e3a03000 	mov	r3, #0
    7b5bc:	e1a08001 	mov	r8, r1
    7b5c0:	e59f174c 	ldr	r1, [pc, #1868]	; 7bd14 <__gxx_personality_v0@@CXXABI_1.3+0x764>
@@ -33,7 +33,7 @@
    7b62c:	e0332002 	eors	r2, r3, r2
    7b630:	1a0001b3 	bne	7bd04 <__gxx_personality_v0@@CXXABI_1.3+0x754>
    7b634:	e28dd054 	add	sp, sp, #84	; 0x54
-   7b638:	ecbd8b02 	vpop	{d8}
+   7b638:	ecbdfb02 	vpop	{d15}
    7b63c:	e8bd8ff0 	pop	{r4, r5, r6, r7, r8, r9, sl, fp, pc}
    7b640:	e15c0003 	cmp	ip, r3
    7b644:	1a0001af 	bne	7bd08 <__gxx_personality_v0@@CXXABI_1.3+0x758>
@@ -317,8 +317,8 @@
    7ba9c:	e5c83007 	strb	r3, [r8, #7]
    7baa0:	e3a03000 	mov	r3, #0
    7baa4:	e28d2024 	add	r2, sp, #36	; 0x24
-   7baa8:	ee087a10 	vmov	s16, r7
-   7baac:	ee082a90 	vmov	s17, r2
+   7baa8:	ee0f7a10 	vmov	s30, r7
+   7baac:	ee0f2a90 	vmov	s31, r2
    7bab0:	e1a07003 	mov	r7, r3
    7bab4:	e3a04000 	mov	r4, #0
    7bab8:	e1a03004 	mov	r3, r4
@@ -398,14 +398,14 @@
    7bbe0:	0a000007 	beq	7bc04 <__gxx_personality_v0@@CXXABI_1.3+0x654>
    7bbe4:	e3580000 	cmp	r8, #0
    7bbe8:	0affffe1 	beq	7bb74 <__gxx_personality_v0@@CXXABI_1.3+0x5c4>
-   7bbec:	ee183a90 	vmov	r3, s17
+   7bbec:	ee1f3a90 	vmov	r3, s31
    7bbf0:	e3a02000 	mov	r2, #0
    7bbf4:	e1a00008 	mov	r0, r8
    7bbf8:	ebffea26 	bl	76498 <__cxa_type_match@plt>
    7bbfc:	e3500000 	cmp	r0, #0
    7bc00:	0affffdb 	beq	7bb74 <__gxx_personality_v0@@CXXABI_1.3+0x5c4>
    7bc04:	e59d3014 	ldr	r3, [sp, #20]
-   7bc08:	ee187a10 	vmov	r7, s16
+   7bc08:	ee1f7a10 	vmov	r7, s30
    7bc0c:	e3130001 	tst	r3, #1
    7bc10:	1afffee8 	bne	7b7b8 <__gxx_personality_v0@@CXXABI_1.3+0x208>
    7bc14:	e59d301c 	ldr	r3, [sp, #28]
@@ -436,7 +436,7 @@
    7bc78:	e3520000 	cmp	r2, #0
    7bc7c:	1afffff5 	bne	7bc58 <__gxx_personality_v0@@CXXABI_1.3+0x6a8>
    7bc80:	e59d3014 	ldr	r3, [sp, #20]
-   7bc84:	ee187a10 	vmov	r7, s16
+   7bc84:	ee1f7a10 	vmov	r7, s30
    7bc88:	e3130001 	tst	r3, #1
    7bc8c:	1afffec9 	bne	7b7b8 <__gxx_personality_v0@@CXXABI_1.3+0x208>
    7bc90:	eaffff4b 	b	7b9c4 <__gxx_personality_v0@@CXXABI_1.3+0x414>
@@ -444,7 +444,7 @@
    7bc98:	eaffffc9 	b	7bbc4 <__gxx_personality_v0@@CXXABI_1.3+0x614>
    7bc9c:	e1a03007 	mov	r3, r7
    7bca0:	e3530000 	cmp	r3, #0
-   7bca4:	ee187a10 	vmov	r7, s16
+   7bca4:	ee1f7a10 	vmov	r7, s30
    7bca8:	0afffe53 	beq	7b5fc <__gxx_personality_v0@@CXXABI_1.3+0x4c>
    7bcac:	e59d3014 	ldr	r3, [sp, #20]
    7bcb0:	e3130001 	tst	r3, #1
@@ -502,4 +502,3 @@
    7bd80:	eaffe0bb 	b	74074 <__cxa_free_exception@plt>
    7bd84:	e514006c 	ldr	r0, [r4, #-108]	; 0xffffff94
    7bd88:	eb0000e1 	bl	7c114 <_ZSt17rethrow_exceptionNSt15__exception_ptr13exception_ptrE@@CXXABI_1.3.3+0xb8>
-

This means that the exception handling framework clobbers d15 instead of d8.  Hopefully, GCC does not use this register for spilling in the test case.  If the test case now passes, it is very likely we are looking at the same bug.  If it crashes in a different way, either d15 is not unused as I expected, or I made a mistake in manual patching.

I've run a mock build of one of the affected packages now, using the patched libstdc++.so.0 when running the tests, and it built and tested successfully.

Created attachment 1524728 [details]
libstdc++.so.6.0.26 built with upstream patch

Tom, would you mind running another test with the attached libstdc++.so.6.0.26 build?  Thanks!

It's using the upstream patch:

https://gcc.gnu.org/bugzilla/attachment.cgi?id=45552&action=edit

(The suggest __cxa_call_unexpected change is not included, it is likely not needed.)

Yes it seems to be fine with that library as well.

Could you please test also https://kojipkgs.fedoraproject.org/packages/gcc/9.0.1/0.2.fc30/armv7hl/ ?

I have done successful scratch builds of both my affected packages (with expensive-optimizations turned back on) with that compiler, so it looks good to me.

This bug appears to have been reported against 'rawhide' during the Fedora 31 development cycle.
Changing version to '31'.

This bug appears to have been reported against 'rawhide' during the Fedora 31 development cycle.
Changing version to 31.

This is long since fixed...