WIP: MDEV-38975 Phase 1 — Promote wide VARCHAR to BLOB for HEAP internal temp tables

include/heap.h

@@ -116,6 +116,9 @@ typedef struct st_hp_keydef		/* Key definition with open */
   uint keysegs;				/* Number of key-segment */
   uint length;				/* Length of key (automatic) */
   uint8 algorithm;			/* HASH / BTREE */
+  my_bool has_blob_seg;			/* Key has HA_BLOB_PART segments */
+  my_bool needs_key_rebuild_from_group_buff; /* GROUP BY key must be rebuilt from group_buff */
+  my_bool blob_data_in_record;		/* BLOB data written to record[0] by heap_store_key_blob_ref */
   HA_KEYSEG *seg;
   HP_BLOCK block;			/* Where keys are saved */
   /*
@@ -131,21 +134,36 @@ typedef struct st_hp_keydef		/* Key definition with open */
   uint (*get_key_length)(struct st_hp_keydef *keydef, const uchar *key);
 } HP_KEYDEF;
 
+static inline my_bool hp_keydef_has_blob_seg(const HP_KEYDEF *keydef)
+{
+  uint j;
+  for (j= 0; j < keydef->keysegs; j++)
+    if (keydef->seg[j].flag & HA_BLOB_PART)
+      return TRUE;
+  return FALSE;
+}
+
+typedef struct st_hp_blob_desc
+{
+  uint offset;       /* Byte offset of blob descriptor within record buffer */
+  uint packlength;   /* 1, 2, 3, or 4: length prefix size */
+} HP_BLOB_DESC;
+
 typedef struct st_heap_share
 {
   HP_BLOCK block;
   HP_KEYDEF  *keydef;
   ulonglong data_length,index_length,max_table_size;
   ulonglong auto_increment;
   ulong min_records,max_records;	/* Params to open */
-  ulong records;			/* records */
+  ulong records;			/* Logical (primary) record count */
   ulong blength;			/* records rounded up to 2^n */
   ulong deleted;			/* Deleted records in database */
   uint key_stat_version;                /* version to indicate insert/delete */
   uint key_version;                     /* Updated on key change */
   uint file_version;                    /* Update on clear */
   uint reclength;			/* Length of one record */
-  uint visible;                         /* Offset to the visible/deleted mark */
+  uint visible;                         /* Offset to the flags byte (active/deleted/continuation) */
   uint changed;
   uint keys,max_key_length;
   uint currently_disabled_keys;    /* saved value from "keys" when disabled */
@@ -156,6 +174,9 @@ typedef struct st_heap_share
   THR_LOCK lock;
   my_bool delete_on_close;
   my_bool internal;                     /* Internal temporary table */
+  HP_BLOB_DESC *blob_descs;    /* Array of blob column descriptors */
+  uint blob_count;             /* Number of blob columns */
+  ulong total_records;         /* All active records (primary + blob continuation) */
   LIST open_list;
   uint auto_key;
   uint auto_key_type;			/* real type of the auto key segment */
@@ -181,6 +202,10 @@ typedef struct st_heap_info
   uint file_version;                    /* Version at scan */
   uint lastkey_len;
   my_bool implicit_emptied;
+  uchar *blob_buff;            /* Reassembly buffer for blob reads */
+  uint32 blob_buff_len;        /* Current allocated size of blob_buff */
+  my_bool has_zerocopy_blobs;  /* Last hp_read_blobs produced zero-copy ptrs */
+  ulong last_hash_of_key;     /* Hash from last hp_search(), reused by hp_search_next() */
   THR_LOCK_DATA lock;
   LIST open_list;
 } HP_INFO;
@@ -204,6 +229,8 @@ typedef struct st_heap_create_info
     open_count to 1. Is only looked at if not internal_table.
   */
   my_bool pin_share;
+  HP_BLOB_DESC *blob_descs;
+  uint blob_count;
 } HP_CREATE_INFO;
 
 	/* Prototypes for heap-functions */

mysql-test/include/mtr_check.sql

@@ -66,7 +66,7 @@ BEGIN
          collation_name, column_type, column_key, extra, column_comment
     FROM INFORMATION_SCHEMA.COLUMNS
       WHERE table_schema='mysql'
-        ORDER BY columns_in_mysql;
+        ORDER BY columns_in_mysql, ordinal_position;
 
   -- Dump all events, there should be none
   SELECT * FROM INFORMATION_SCHEMA.EVENTS;

mysql-test/main/blob_sj_test.result

@@ -0,0 +1,29 @@
+set optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
+set @blob_len = 16;
+set @prefix_len = 6;
+set @suffix_len = @blob_len - @prefix_len;
+create table t1 (a1 blob(16), a2 blob(16));
+create table t2 (b1 blob(16), b2 blob(16));
+insert into t1 values
+(concat('1 - 00', repeat('x', @suffix_len)), concat('2 - 00', repeat('x', @suffix_len)));
+insert into t1 values
+(concat('1 - 01', repeat('x', @suffix_len)), concat('2 - 01', repeat('x', @suffix_len)));
+insert into t1 values
+(concat('1 - 02', repeat('x', @suffix_len)), concat('2 - 02', repeat('x', @suffix_len)));
+insert into t2 values
+(concat('1 - 01', repeat('x', @suffix_len)), concat('2 - 01', repeat('x', @suffix_len)));
+insert into t2 values
+(concat('1 - 02', repeat('x', @suffix_len)), concat('2 - 02', repeat('x', @suffix_len)));
+insert into t2 values
+(concat('1 - 03', repeat('x', @suffix_len)), concat('2 - 03', repeat('x', @suffix_len)));
+explain extended select left(a1,7), left(a2,7) from t1 where a1 in (select b1 from t2 where b1 > '0');
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where
+2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where
+Warnings:
+Note	1003	/* select#1 */ select left(`test`.`t1`.`a1`,7) AS `left(a1,7)`,left(`test`.`t1`.`a2`,7) AS `left(a2,7)` from `test`.`t1` where <expr_cache><`test`.`t1`.`a1`>(<in_optimizer>(`test`.`t1`.`a1`,<exists>(/* select#2 */ select `test`.`t2`.`b1` from `test`.`t2` where `test`.`t2`.`b1` > '0' and <cache>(`test`.`t1`.`a1`) = `test`.`t2`.`b1`)))
+select left(a1,7), left(a2,7) from t1 where a1 in (select b1 from t2 where b1 > '0');
+left(a1,7)	left(a2,7)
+1 - 01x	2 - 01x
+1 - 02x	2 - 02x
+drop table t1, t2;

mysql-test/main/blob_sj_test.test

@@ -0,0 +1,26 @@
+set optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
+set @blob_len = 16;
+set @prefix_len = 6;
+set @suffix_len = @blob_len - @prefix_len;
+
+create table t1 (a1 blob(16), a2 blob(16));
+create table t2 (b1 blob(16), b2 blob(16));
+
+insert into t1 values
+(concat('1 - 00', repeat('x', @suffix_len)), concat('2 - 00', repeat('x', @suffix_len)));
+insert into t1 values
+(concat('1 - 01', repeat('x', @suffix_len)), concat('2 - 01', repeat('x', @suffix_len)));
+insert into t1 values
+(concat('1 - 02', repeat('x', @suffix_len)), concat('2 - 02', repeat('x', @suffix_len)));
+
+insert into t2 values
+(concat('1 - 01', repeat('x', @suffix_len)), concat('2 - 01', repeat('x', @suffix_len)));
+insert into t2 values
+(concat('1 - 02', repeat('x', @suffix_len)), concat('2 - 02', repeat('x', @suffix_len)));
+insert into t2 values
+(concat('1 - 03', repeat('x', @suffix_len)), concat('2 - 03', repeat('x', @suffix_len)));
+
+explain extended select left(a1,7), left(a2,7) from t1 where a1 in (select b1 from t2 where b1 > '0');
+select left(a1,7), left(a2,7) from t1 where a1 in (select b1 from t2 where b1 > '0');
+
+drop table t1, t2;

mysql-test/main/count_distinct.result

@@ -31,34 +31,34 @@ isbn	city	libname	a
 000	New York	New York Public Libra	6
 001	New York	NYC Lib	1
 006	San Fran	San Fransisco Public	1
-select t2.isbn,city,t1.libname,count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct t1.libname) > 1;
-isbn	city	libname	a
+select min(t2.isbn),city,min(t1.libname),count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct t1.libname) > 1;
+min(t2.isbn)	city	min(t1.libname)	a
 007	Berkeley	Berkeley Public1	2
 000	New York	New York Public Libra	2
-select t2.isbn,city,t1.libname,count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct concat(t1.libname,'a')) > 1;
-isbn	city	libname	a
+select min(t2.isbn),city,min(t1.libname),count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct concat(t1.libname,'a')) > 1;
+min(t2.isbn)	city	min(t1.libname)	a
 007	Berkeley	Berkeley Public1	2
 000	New York	New York Public Libra	2
-select t2.isbn,city,@bar:=t1.libname,count(distinct t1.libname) as a
+select min(t2.isbn),city,@bar:=min(t1.libname),count(distinct t1.libname) as a
 from t3 left join t1 on t3.libname=t1.libname left join t2
 on t3.isbn=t2.isbn group by city having count(distinct
 t1.libname) > 1;
-isbn	city	@bar:=t1.libname	a
+min(t2.isbn)	city	@bar:=min(t1.libname)	a
 007	Berkeley	Berkeley Public1	2
 000	New York	New York Public Libra	2
 SELECT @bar;
 @bar
-Berkeley Public2
-select t2.isbn,city,concat(@bar:=t1.libname),count(distinct t1.libname) as a
+New York Public Libra
+select min(t2.isbn),city,concat(@bar:=min(t1.libname)),count(distinct t1.libname) as a
 from t3 left join t1 on t3.libname=t1.libname left join t2
 on t3.isbn=t2.isbn group by city having count(distinct
 t1.libname) > 1;
-isbn	city	concat(@bar:=t1.libname)	a
+min(t2.isbn)	city	concat(@bar:=min(t1.libname))	a
 007	Berkeley	Berkeley Public1	2
 000	New York	New York Public Libra	2
 SELECT @bar;
 @bar
-Berkeley Public2
+New York Public Libra
 drop table t1, t2, t3;
 create table t1 (f1 int);
 insert into t1 values (1);

mysql-test/main/count_distinct.test

@@ -29,25 +29,19 @@ insert into t1 values ('Berkeley Public1','Berkeley');
 insert into t1 values ('Berkeley Public2','Berkeley');
 insert into t1 values ('NYC Lib','New York');
 select t2.isbn,city,t1.libname,count(t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city,t1.libname;
-select t2.isbn,city,t1.libname,count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct t1.libname) > 1;
-select t2.isbn,city,t1.libname,count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct concat(t1.libname,'a')) > 1;
+select min(t2.isbn),city,min(t1.libname),count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct t1.libname) > 1;
+select min(t2.isbn),city,min(t1.libname),count(distinct t1.libname) as a from t3 left join t1 on t3.libname=t1.libname left join t2 on t3.isbn=t2.isbn group by city having count(distinct concat(t1.libname,'a')) > 1;
 
-select t2.isbn,city,@bar:=t1.libname,count(distinct t1.libname) as a
+select min(t2.isbn),city,@bar:=min(t1.libname),count(distinct t1.libname) as a
        from t3 left join t1 on t3.libname=t1.libname left join t2
        on t3.isbn=t2.isbn group by city having count(distinct
        t1.libname) > 1;
-#
-# Wrong result, see bug#49872
-# 
 SELECT @bar;
 
-select t2.isbn,city,concat(@bar:=t1.libname),count(distinct t1.libname) as a
+select min(t2.isbn),city,concat(@bar:=min(t1.libname)),count(distinct t1.libname) as a
        from t3 left join t1 on t3.libname=t1.libname left join t2
        on t3.isbn=t2.isbn group by city having count(distinct
        t1.libname) > 1;
-#
-# Wrong result, see bug#49872
-# 
 SELECT @bar;
 
 drop table t1, t2, t3;

mysql-test/main/create.result

@@ -30,10 +30,7 @@ Note	1051	Unknown table 'test.t1,test.t2'
 create table t1 (b char(0) not null, index(b));
 ERROR 42000: The storage engine MyISAM can't index column `b`
 create table t1 (a int not null,b text) engine=heap;
-ERROR 42000: Storage engine MEMORY doesn't support BLOB/TEXT columns
-drop table if exists t1;
-Warnings:
-Note	1051	Unknown table 'test.t1'
+drop table t1;
 create table t1 (ordid int(8) not null auto_increment, ord  varchar(50) not null, primary key (ord,ordid)) engine=heap;
 ERROR 42000: Incorrect table definition; there can be only one auto column and it must be defined as a key
 create table not_existing_database.test (a int);
@@ -1072,12 +1069,12 @@ show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
   `ID` bigint(4) NOT NULL,
-  `USER` varchar(128) NOT NULL,
-  `HOST` varchar(64) NOT NULL,
-  `DB` varchar(64),
-  `COMMAND` varchar(16) NOT NULL,
+  `USER` longtext NOT NULL,
+  `HOST` longtext NOT NULL,
+  `DB` longtext,
+  `COMMAND` longtext NOT NULL,
   `TIME` int(7) NOT NULL,
-  `STATE` varchar(64),
+  `STATE` longtext,
   `INFO` longtext,
   `TIME_MS` decimal(22,3) NOT NULL,
   `STAGE` tinyint(2) NOT NULL,
@@ -1089,19 +1086,19 @@ t1	CREATE TABLE `t1` (
   `QUERY_ID` bigint(4) NOT NULL,
   `INFO_BINARY` blob,
   `TID` bigint(4) NOT NULL
-)  DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
+) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
 drop table t1;
 create temporary table t1 like information_schema.processlist;
 show create table t1;
 Table	Create Table
 t1	CREATE TEMPORARY TABLE `t1` (
   `ID` bigint(4) NOT NULL,
-  `USER` varchar(128) NOT NULL,
-  `HOST` varchar(64) NOT NULL,
-  `DB` varchar(64),
-  `COMMAND` varchar(16) NOT NULL,
+  `USER` longtext NOT NULL,
+  `HOST` longtext NOT NULL,
+  `DB` longtext,
+  `COMMAND` longtext NOT NULL,
   `TIME` int(7) NOT NULL,
-  `STATE` varchar(64),
+  `STATE` longtext,
   `INFO` longtext,
   `TIME_MS` decimal(22,3) NOT NULL,
   `STAGE` tinyint(2) NOT NULL,
@@ -1113,15 +1110,15 @@ t1	CREATE TEMPORARY TABLE `t1` (
   `QUERY_ID` bigint(4) NOT NULL,
   `INFO_BINARY` blob,
   `TID` bigint(4) NOT NULL
-)  DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
+) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
 drop table t1;
 create table t1 like information_schema.character_sets;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `CHARACTER_SET_NAME` varchar(32) NOT NULL,
-  `DEFAULT_COLLATE_NAME` varchar(64) NOT NULL,
-  `DESCRIPTION` varchar(60) NOT NULL,
+  `CHARACTER_SET_NAME` longtext NOT NULL,
+  `DEFAULT_COLLATE_NAME` longtext NOT NULL,
+  `DESCRIPTION` longtext NOT NULL,
   `MAXLEN` bigint(3) NOT NULL
 ) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
 drop table t1;

mysql-test/main/create.test

@@ -30,9 +30,8 @@ create table t2 select auto+1 from t1;
 drop table if exists t1,t2;
 --error ER_WRONG_KEY_COLUMN
 create table t1 (b char(0) not null, index(b));
---error ER_TABLE_CANT_HANDLE_BLOB
 create table t1 (a int not null,b text) engine=heap;
-drop table if exists t1;
+drop table t1;
 
 --error ER_WRONG_AUTO_KEY
 create table t1 (ordid int(8) not null auto_increment, ord  varchar(50) not null, primary key (ord,ordid)) engine=heap;

mysql-test/main/cte_recursive.test

@@ -3212,6 +3212,8 @@ show create table t2;
 --eval insert ignore into t2 $query;
 drop table t2;
 set @@sql_mode="";
+# Rows with identical (level, mid) due to overflow have non-deterministic order
+--sorted_result
 --eval $query
 --eval create table t2 as $query;
 show create table t2;

mysql-test/main/ctype_binary.result

@@ -2382,7 +2382,7 @@ insert into t1 values (1);
 create view v1(a) as select concat(a) from t1;
 show columns from v1;
 Field	Type	Null	Key	Default	Extra
-a	varbinary(64)	YES		NULL	
+a	tinyblob	YES		NULL	
 select hex(a) from v1;
 hex(a)
 0000000000000001

mysql-test/main/ctype_cp1251.result

@@ -2794,7 +2794,7 @@ insert into t1 values (1);
 create view v1(a) as select concat(a) from t1;
 show columns from v1;
 Field	Type	Null	Key	Default	Extra
-a	varbinary(64)	YES		NULL	
+a	tinyblob	YES		NULL	
 select hex(a) from v1;
 hex(a)
 0000000000000001

mysql-test/main/ctype_latin1.result

@@ -3103,7 +3103,7 @@ insert into t1 values (1);
 create view v1(a) as select concat(a) from t1;
 show columns from v1;
 Field	Type	Null	Key	Default	Extra
-a	varbinary(64)	YES		NULL	
+a	tinyblob	YES		NULL	
 select hex(a) from v1;
 hex(a)
 0000000000000001